from typing import List
import numpy as np
from catboost import CatBoostClassifier
from lightgbm import LGBMClassifier
from sklearn.datasets import make_classification, make_regression
from sklearn.linear_model import RidgeClassifier, Ridge
from sklearn.metrics import roc_auc_score, r2_score
from sklearn.model_selection import train_test_split, KFold
from nyaggle.experiment import autoprep_gbdt
from nyaggle.testing import make_classification_df
from nyaggle.validation import cross_validate, Take
def test_cv_sklean_binary():
X, y = make_classification(n_samples=1024, n_features=20, class_sep=0.98, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
model = RidgeClassifier(alpha=1.0)
pred_oof, pred_test, scores, _ = cross_validate(model, X_train, y_train, X_test, cv=5, eval_func=roc_auc_score)
assert len(scores) == 5 + 1
assert scores[-1] >= 0.85 # overall auc
assert roc_auc_score(y_train, pred_oof) == scores[-1]
assert roc_auc_score(y_test, pred_test) >= 0.85 # test score
def test_cv_sklean_regression():
X, y = make_regression(n_samples=1024, n_features=20, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
model = Ridge(alpha=1.0)
pred_oof, pred_test, scores, _ = cross_validate(model, X_train, y_train, X_test, cv=5, eval_func=r2_score)
print(scores)
assert len(scores) == 5 + 1
assert scores[-1] >= 0.95 # overall r2
assert r2_score(y_train, pred_oof) == scores[-1]
assert r2_score(y_test, pred_test) >= 0.95 # test r2
def test_cv_lgbm():
X, y = make_classification(n_samples=1024, n_features=20, class_sep=0.98, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
models = [LGBMClassifier(n_estimators=300) for _ in range(5)]
pred_oof, pred_test, scores, importance = cross_validate(models, X_train, y_train, X_test, cv=5,
eval_func=roc_auc_score,
fit_params={'early_stopping_rounds': 200})
print(scores)
assert len(scores) == 5 + 1
assert scores[-1] >= 0.85 # overall roc_auc
assert roc_auc_score(y_train, pred_oof) == scores[-1]
assert roc_auc_score(y_test, pred_test) >= 0.85 # test roc_auc
assert roc_auc_score(y, models[0].predict_proba(X)[:, 1]) >= 0.85 # make sure models are trained
assert len(importance) == 5
assert list(importance[0].columns) == ['feature', 'importance']
assert len(importance[0]) == 20
def test_cv_lgbm_df():
X, y = make_classification_df(n_samples=1024, n_num_features=20, n_cat_features=1, class_sep=0.98, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
models = [LGBMClassifier(n_estimators=300) for _ in range(5)]
pred_oof, pred_test, scores, importance = cross_validate(models, X_train, y_train, X_test, cv=5,
eval_func=roc_auc_score)
print(scores)
assert len(scores) == 5 + 1
assert scores[-1] >= 0.85 # overall roc_auc
assert roc_auc_score(y_train, pred_oof) == scores[-1]
assert roc_auc_score(y_test, pred_test) >= 0.85 # test roc_auc
assert roc_auc_score(y_test, models[0].predict_proba(X_test)[:, 1]) >= 0.85 # make sure models are trained
assert len(importance) == 5
assert list(importance[0].columns) == ['feature', 'importance']
assert len(importance[0]) == 20 + 1
assert models[0].booster_.num_trees() < 300 # making sure early stopping worked
def test_cv_cat_df():
X, y = make_classification_df(n_samples=1024, n_num_features=20, n_cat_features=1, class_sep=0.98, random_state=0)
X, _ = autoprep_gbdt('cat', X, None)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
models = [CatBoostClassifier(n_estimators=300) for _ in range(5)]
pred_oof, pred_test, scores, importance = cross_validate(models, X_train, y_train, X_test, cv=5,
eval_func=roc_auc_score,
fit_params={'cat_features': ['cat_0']})
print(scores)
assert len(scores) == 5 + 1
assert scores[-1] >= 0.85 # overall roc_auc
assert roc_auc_score(y_train, pred_oof) == scores[-1]
assert roc_auc_score(y_test, pred_test) >= 0.85 # test roc_auc
assert roc_auc_score(y_test, models[0].predict_proba(X_test)[:, 1]) >= 0.85 # make sure models are trained
assert len(importance) == 5
assert list(importance[0].columns) == ['feature', 'importance']
assert len(importance[0]) == 20 + 1
assert models[0].tree_count_ < 300 # making sure early stopping worked
def test_cv_partial_evaluate():
X, y = make_classification(n_samples=1024, n_features=20, class_sep=0.98, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
model = RidgeClassifier(alpha=1.0)
n = 0
def _fold_count(*args):
nonlocal n
n += 1
cv = Take(2, KFold(5))
pred_oof, pred_test, scores, _ = cross_validate(model, X_train, y_train, X_test, cv=cv, eval_func=roc_auc_score,
on_each_fold=_fold_count)
assert len(scores) == 2 + 1
assert scores[-1] >= 0.8 # overall auc
assert n == 2
def test_fit_params_callback():
X, y = make_classification(n_samples=1024, n_features=20, class_sep=0.98, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0)
models = [LGBMClassifier(n_estimators=300) for _ in range(5)]
sample_weights = np.random.randint(1, 10, size=len(X_train))
sample_weights = sample_weights / sample_weights.sum()
def fit_params(n: int, train_index: List[int], valid_index: List[int]):
return {
'early_stopping_rounds': 100,
'sample_weight': list(sample_weights[train_index]),
'eval_sample_weight': [list(sample_weights[valid_index])]
}
result_w_weight = cross_validate(models, X_train, y_train, X_test, cv=5,
eval_func=roc_auc_score, fit_params=fit_params)
result_wo_weight = cross_validate(models, X_train, y_train, X_test, cv=5,
eval_func=roc_auc_score, fit_params={'early_stopping_rounds': 50})
assert result_w_weight.scores[-1] != result_wo_weight.scores[-1]
