6. Classification

6.1. Generate data

import numpy as np
from numpy.random import normal
from scipy.stats import binom
from collections import namedtuple

Data = namedtuple('Data', 'X y')

np.random.seed(37)

def get_data(N=10000, M=10):
    X = np.hstack([normal(0.0, 1.0, N).reshape(N, 1) for _ in range(M)])

    w = np.array([w + 1.0 for w in range(M)])
    z = np.dot(X, w) + normal(0.0, 0.2, N)
    p = 1.0 / (1.0 + np.exp(-z))
    y = binom.rvs(1, p)

    return Data(X, y)

# training
T = get_data()

# validation
V = get_data(N=1000)

6.2. Types of classifiers

6.2.1. Logistic regression

from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(penalty='l1', solver='liblinear', fit_intercept=False)
lr.fit(T.X, T.y)

LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=False,
                   intercept_scaling=1, l1_ratio=None, max_iter=100,
                   multi_class='warn', n_jobs=None, penalty='l1',
                   random_state=None, solver='liblinear', tol=0.0001, verbose=0,
                   warm_start=False)

6.2.2. Gaussian Naive Bayes

from sklearn.naive_bayes import GaussianNB

nb = GaussianNB()
nb.fit(T.X, T.y)

GaussianNB(priors=None, var_smoothing=1e-09)

6.2.3. Linear Discriminant Analysis

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

lda = LinearDiscriminantAnalysis()
lda.fit(T.X, T.y)

LinearDiscriminantAnalysis(n_components=None, priors=None, shrinkage=None,
                           solver='svd', store_covariance=False, tol=0.0001)

6.2.4. Quadratic Discriminant Analysis

from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

qda = QuadraticDiscriminantAnalysis()
qda.fit(T.X, T.y)

QuadraticDiscriminantAnalysis(priors=None, reg_param=0.0,
                              store_covariance=False, tol=0.0001)

6.2.5. Neural network, Multi-Layer Perceptron (MLP)

from sklearn.neural_network import MLPClassifier

mlp = MLPClassifier(max_iter=1000, alpha=0.01)
mlp.fit(T.X, T.y)

MLPClassifier(activation='relu', alpha=0.01, batch_size='auto', beta_1=0.9,
              beta_2=0.999, early_stopping=False, epsilon=1e-08,
              hidden_layer_sizes=(100,), learning_rate='constant',
              learning_rate_init=0.001, max_iter=1000, momentum=0.9,
              n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,
              random_state=None, shuffle=True, solver='adam', tol=0.0001,
              validation_fraction=0.1, verbose=False, warm_start=False)

6.2.6. Decision tree

from sklearn.tree import DecisionTreeClassifier

dt = DecisionTreeClassifier()
dt.fit(T.X, T.y)

DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,
                       max_features=None, max_leaf_nodes=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, presort=False,
                       random_state=None, splitter='best')

6.2.7. Linear Support Vector Machine (SVM)

from sklearn.svm import NuSVC

svm = NuSVC(gamma='auto', probability=True, random_state=37)
svm.fit(T.X, T.y)

NuSVC(cache_size=200, class_weight=None, coef0=0.0,
      decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',
      max_iter=-1, nu=0.5, probability=True, random_state=37, shrinking=True,
      tol=0.001, verbose=False)

6.2.8. Stochastic gradient descient (SGD)

from sklearn.linear_model import SGDClassifier

sgd = SGDClassifier(loss='log')
sgd.fit(T.X, T.y)

SGDClassifier(alpha=0.0001, average=False, class_weight=None,
              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,
              l1_ratio=0.15, learning_rate='optimal', loss='log', max_iter=1000,
              n_iter_no_change=5, n_jobs=None, penalty='l2', power_t=0.5,
              random_state=None, shuffle=True, tol=0.001,
              validation_fraction=0.1, verbose=0, warm_start=False)

6.2.9. Random Forest

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators=100)
rf.fit(T.X, T.y)

RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
                       max_depth=None, max_features='auto', max_leaf_nodes=None,
                       min_impurity_decrease=0.0, min_impurity_split=None,
                       min_samples_leaf=1, min_samples_split=2,
                       min_weight_fraction_leaf=0.0, n_estimators=100,
                       n_jobs=None, oob_score=False, random_state=None,
                       verbose=0, warm_start=False)

6.2.10. AdaBoost

from sklearn.ensemble import AdaBoostClassifier

ab = AdaBoostClassifier()
ab.fit(T.X, T.y)

AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None, learning_rate=1.0,
                   n_estimators=50, random_state=None)

6.3. Performance

import pandas as pd
from sklearn.metrics import accuracy_score, f1_score, jaccard_score, matthews_corrcoef, \
    precision_score, recall_score, \
    brier_score_loss, log_loss, hamming_loss, hinge_loss, zero_one_loss, \
    roc_auc_score, average_precision_score

def get_scoring_functions():
    cfuncs = [accuracy_score, f1_score, jaccard_score, matthews_corrcoef]
    pfuncs = [brier_score_loss, log_loss, hinge_loss,
              roc_auc_score, average_precision_score]
    return cfuncs, pfuncs

def get_tuple_cols():
    cfuncs, pfuncs = get_scoring_functions()
    return ['model'] + [f.__name__ for f in cfuncs] + [f.__name__ for f in pfuncs]

def get_scores(model_name, y_true, y_preds, y_probs):
    cfuncs, pfuncs = get_scoring_functions()

    cscores = {f.__name__: f(y_true, y_preds) for f in cfuncs}
    pscores = {f.__name__: f(y_true, y_probs) for f in pfuncs}

    d = {**cscores, **pscores}
    d['model'] = model_name

    return tuple([d[c] for c in get_tuple_cols()])

models = [lr, nb, lda, qda, mlp, dt, svm, sgd, rf, ab]
model_names = [type(m).__name__ for m in models]

y_preds = {type(model).__name__: model.predict(V.X) for model in models}
y_probs = {type(model).__name__: model.predict_proba(V.X)[:,1] for model in models}

scores = [get_scores(name, V.y, y_preds[name], y_probs[name]) for name in model_names]
df = pd.DataFrame(scores, columns=get_tuple_cols())
df

	model	accuracy_score	f1_score	jaccard_score	matthews_corrcoef	brier_score_loss	log_loss	hinge_loss	roc_auc_score	average_precision_score
0	LogisticRegression	0.978	0.976842	0.954733	0.956086	0.016632	0.054495	0.569966	0.998527	0.998359
1	GaussianNB	0.968	0.966030	0.934292	0.935792	0.079076	0.294971	0.770754	0.997130	0.996774
2	LinearDiscriminantAnalysis	0.971	0.969442	0.940695	0.942007	0.038022	0.149966	0.653551	0.998378	0.998193
3	QuadraticDiscriminantAnalysis	0.978	0.976842	0.954733	0.956086	0.038260	0.150470	0.653850	0.998246	0.998042
4	MLPClassifier	0.971	0.969506	0.940816	0.942097	0.019656	0.062244	0.569476	0.997579	0.997283
5	DecisionTreeClassifier	0.814	0.803797	0.671958	0.627096	0.186000	6.424290	0.716000	0.813810	0.735136
6	NuSVC	0.969	0.967334	0.936735	0.937996	0.020646	0.067079	0.578227	0.997543	0.997283
7	SGDClassifier	0.973	0.971307	0.944215	0.945813	0.018218	0.061125	0.576176	0.998270	0.998065
8	RandomForestClassifier	0.926	0.921610	0.854617	0.851564	0.076666	0.271507	0.744460	0.981208	0.978537
9	AdaBoostClassifier	0.921	0.916754	0.846304	0.841738	0.228336	0.649722	1.007489	0.979775	0.978109