Skip to content

Evaluators

siapy.utils.evaluators

ScorerFuncType module-attribute 

ScorerFuncType = Callable[[BaseEstimator, ArrayLike2dType, ArrayLike1dType], float]

cross_validation 

cross_validation(model: BaseEstimator, X: ArrayLike2dType, y: ArrayLike1dType, X_val: ArrayLike2dType | None = None, y_val: ArrayLike1dType | None = None, *, groups: ArrayLike1dType | None = None, scoring: str | ScorerFuncType | None = None, cv: int | BaseCrossValidator | Iterable | None = None, n_jobs: int | None = 1, verbose: int = 0, fit_params: dict[str, Any] | None = None, pre_dispatch: int | str = 1, error_score: Literal['raise'] | int = 0) -> float
PARAMETER DESCRIPTION
model

TYPE: BaseEstimator

X

TYPE: ArrayLike2dType

y

TYPE: ArrayLike1dType

X_val

TYPE: ArrayLike2dType | None DEFAULT: None

y_val

TYPE: ArrayLike1dType | None DEFAULT: None

groups

TYPE: ArrayLike1dType | None DEFAULT: None

scoring

TYPE: str | ScorerFuncType | None DEFAULT: None

cv

TYPE: int | BaseCrossValidator | Iterable | None DEFAULT: None

n_jobs

TYPE: int | None DEFAULT: 1

verbose

TYPE: int DEFAULT: 0

fit_params

TYPE: dict[str, Any] | None DEFAULT: None

pre_dispatch

TYPE: int | str DEFAULT: 1

error_score

TYPE: Literal['raise'] | int DEFAULT: 0

Source code in siapy/utils/evaluators.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
def cross_validation(
    model: BaseEstimator,
    X: ArrayLike2dType,
    y: ArrayLike1dType,
    X_val: Annotated[ArrayLike2dType | None, "Not used, only for compatibility"] = None,
    y_val: Annotated[ArrayLike1dType | None, "Not used, only for compatibility"] = None,
    *,
    groups: ArrayLike1dType | None = None,
    scoring: str | ScorerFuncType | None = None,
    cv: int | BaseCrossValidator | Iterable | None = None,
    n_jobs: int | None = 1,
    verbose: int = 0,
    fit_params: dict[str, Any] | None = None,
    pre_dispatch: int | str = 1,
    error_score: Literal["raise"] | int = 0,
) -> float:
    if X_val is not None or y_val is not None:
        logger.info(
            "Specification of X_val and y_val is redundant for cross_validation."
            "These parameters are ignored."
        )
    check_model_prediction_methods(model)
    score = cross_val_score(
        estimator=model,
        X=X,  # type: ignore
        y=y,
        groups=groups,
        scoring=scoring,
        cv=cv,
        n_jobs=n_jobs,
        verbose=verbose,
        fit_params=fit_params,
        pre_dispatch=pre_dispatch,
        error_score=error_score,
    )
    return score.mean()

hold_out_validation 

hold_out_validation(model: BaseEstimator, X: ArrayLike2dType, y: ArrayLike1dType, X_val: ArrayLike2dType | None = None, y_val: ArrayLike1dType | None = None, *, scoring: str | ScorerFuncType | None = None, test_size: float | None = 0.2, random_state: int | None = None, shuffle: bool = True, stratify: ndarray | None = None) -> float
PARAMETER DESCRIPTION
model

TYPE: BaseEstimator

X

TYPE: ArrayLike2dType

y

TYPE: ArrayLike1dType

X_val

TYPE: ArrayLike2dType | None DEFAULT: None

y_val

TYPE: ArrayLike1dType | None DEFAULT: None

scoring

TYPE: str | ScorerFuncType | None DEFAULT: None

test_size

TYPE: float | None DEFAULT: 0.2

random_state

TYPE: int | None DEFAULT: None

shuffle

TYPE: bool DEFAULT: True

stratify

TYPE: ndarray | None DEFAULT: None

Source code in siapy/utils/evaluators.py 
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def hold_out_validation(
    model: BaseEstimator,
    X: ArrayLike2dType,
    y: ArrayLike1dType,
    X_val: ArrayLike2dType | None = None,
    y_val: ArrayLike1dType | None = None,
    *,
    scoring: str | ScorerFuncType | None = None,
    test_size: float | None = 0.2,
    random_state: int | None = None,
    shuffle: bool = True,
    stratify: np.ndarray | None = None,
) -> float:
    if X_val is not None and y_val is not None:
        x_train, x_test, y_train, y_test = X, X_val, y, y_val
    elif X_val is not None or y_val is not None:
        raise InvalidInputError(
            input_value={"X_val": X_val, "y_val": y_val},
            message="To manually define validation set, both X_val and y_val must be specified.",
        )
    else:
        x_train, x_test, y_train, y_test = train_test_split(
            X,
            y,
            test_size=test_size,
            random_state=random_state,
            shuffle=shuffle,
            stratify=stratify,
        )
    check_model_prediction_methods(model)
    model.fit(x_train, y_train)  # type: ignore

    if scoring:
        if isinstance(scoring, str):
            scoring_func = get_scorer(scoring)
        else:
            scoring_func = scoring
        score = scoring_func(model, x_test, y_test)
    else:
        score = model.score(x_test, y_test)  # type: ignore
    return score