# 6. Classification

## 6.1. Generate data

[1]:

import numpy as np
from numpy.random import normal
from scipy.stats import binom
from collections import namedtuple

Data = namedtuple('Data', 'X y')

np.random.seed(37)

def get_data(N=10000, M=10):
X = np.hstack([normal(0.0, 1.0, N).reshape(N, 1) for _ in range(M)])

w = np.array([w + 1.0 for w in range(M)])
z = np.dot(X, w) + normal(0.0, 0.2, N)
p = 1.0 / (1.0 + np.exp(-z))
y = binom.rvs(1, p)

return Data(X, y)

# training
T = get_data()

# validation
V = get_data(N=1000)


## 6.2. Types of classifiers

### 6.2.1. Logistic regression

[2]:

from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(penalty='l1', solver='liblinear', fit_intercept=False)
lr.fit(T.X, T.y)

[2]:

LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=False,
intercept_scaling=1, l1_ratio=None, max_iter=100,
multi_class='warn', n_jobs=None, penalty='l1',
random_state=None, solver='liblinear', tol=0.0001, verbose=0,
warm_start=False)


### 6.2.2. Gaussian Naive Bayes

[3]:

from sklearn.naive_bayes import GaussianNB

nb = GaussianNB()
nb.fit(T.X, T.y)

[3]:

GaussianNB(priors=None, var_smoothing=1e-09)


### 6.2.3. Linear Discriminant Analysis

[4]:

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

lda = LinearDiscriminantAnalysis()
lda.fit(T.X, T.y)

[4]:

LinearDiscriminantAnalysis(n_components=None, priors=None, shrinkage=None,
solver='svd', store_covariance=False, tol=0.0001)


[5]:

from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

qda.fit(T.X, T.y)

[5]:

QuadraticDiscriminantAnalysis(priors=None, reg_param=0.0,
store_covariance=False, tol=0.0001)


### 6.2.5. Neural network, Multi-Layer Perceptron (MLP)

[6]:

from sklearn.neural_network import MLPClassifier

mlp = MLPClassifier(max_iter=1000, alpha=0.01)
mlp.fit(T.X, T.y)

[6]:

MLPClassifier(activation='relu', alpha=0.01, batch_size='auto', beta_1=0.9,
beta_2=0.999, early_stopping=False, epsilon=1e-08,
hidden_layer_sizes=(100,), learning_rate='constant',
learning_rate_init=0.001, max_iter=1000, momentum=0.9,
n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,
validation_fraction=0.1, verbose=False, warm_start=False)


### 6.2.6. Decision tree

[7]:

from sklearn.tree import DecisionTreeClassifier

dt = DecisionTreeClassifier()
dt.fit(T.X, T.y)

[7]:

DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,
max_features=None, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, presort=False,
random_state=None, splitter='best')


### 6.2.7. Linear Support Vector Machine (SVM)

[8]:

from sklearn.svm import NuSVC

svm = NuSVC(gamma='auto', probability=True, random_state=37)
svm.fit(T.X, T.y)

[8]:

NuSVC(cache_size=200, class_weight=None, coef0=0.0,
decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',
max_iter=-1, nu=0.5, probability=True, random_state=37, shrinking=True,
tol=0.001, verbose=False)


### 6.2.8. Stochastic gradient descient (SGD)

[9]:

from sklearn.linear_model import SGDClassifier

sgd = SGDClassifier(loss='log')
sgd.fit(T.X, T.y)

[9]:

SGDClassifier(alpha=0.0001, average=False, class_weight=None,
early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,
l1_ratio=0.15, learning_rate='optimal', loss='log', max_iter=1000,
n_iter_no_change=5, n_jobs=None, penalty='l2', power_t=0.5,
random_state=None, shuffle=True, tol=0.001,
validation_fraction=0.1, verbose=0, warm_start=False)


### 6.2.9. Random Forest

[10]:

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators=100)
rf.fit(T.X, T.y)

[10]:

RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
max_depth=None, max_features='auto', max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=100,
n_jobs=None, oob_score=False, random_state=None,
verbose=0, warm_start=False)


[11]:

from sklearn.ensemble import AdaBoostClassifier

ab.fit(T.X, T.y)

[11]:

AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None, learning_rate=1.0,
n_estimators=50, random_state=None)


## 6.3. Performance

[12]:

import pandas as pd
from sklearn.metrics import accuracy_score, f1_score, jaccard_score, matthews_corrcoef, \
precision_score, recall_score, \
brier_score_loss, log_loss, hamming_loss, hinge_loss, zero_one_loss, \
roc_auc_score, average_precision_score

def get_scoring_functions():
cfuncs = [accuracy_score, f1_score, jaccard_score, matthews_corrcoef]
pfuncs = [brier_score_loss, log_loss, hinge_loss,
roc_auc_score, average_precision_score]
return cfuncs, pfuncs

def get_tuple_cols():
cfuncs, pfuncs = get_scoring_functions()
return ['model'] + [f.__name__ for f in cfuncs] + [f.__name__ for f in pfuncs]

def get_scores(model_name, y_true, y_preds, y_probs):
cfuncs, pfuncs = get_scoring_functions()

cscores = {f.__name__: f(y_true, y_preds) for f in cfuncs}
pscores = {f.__name__: f(y_true, y_probs) for f in pfuncs}

d = {**cscores, **pscores}
d['model'] = model_name

return tuple([d[c] for c in get_tuple_cols()])

models = [lr, nb, lda, qda, mlp, dt, svm, sgd, rf, ab]
model_names = [type(m).__name__ for m in models]

y_preds = {type(model).__name__: model.predict(V.X) for model in models}
y_probs = {type(model).__name__: model.predict_proba(V.X)[:,1] for model in models}

scores = [get_scores(name, V.y, y_preds[name], y_probs[name]) for name in model_names]
df = pd.DataFrame(scores, columns=get_tuple_cols())
df

[12]:

model accuracy_score f1_score jaccard_score matthews_corrcoef brier_score_loss log_loss hinge_loss roc_auc_score average_precision_score
0 LogisticRegression 0.978 0.976842 0.954733 0.956086 0.016632 0.054495 0.569966 0.998527 0.998359
1 GaussianNB 0.968 0.966030 0.934292 0.935792 0.079076 0.294971 0.770754 0.997130 0.996774
2 LinearDiscriminantAnalysis 0.971 0.969442 0.940695 0.942007 0.038022 0.149966 0.653551 0.998378 0.998193
3 QuadraticDiscriminantAnalysis 0.978 0.976842 0.954733 0.956086 0.038260 0.150470 0.653850 0.998246 0.998042
4 MLPClassifier 0.971 0.969506 0.940816 0.942097 0.019656 0.062244 0.569476 0.997579 0.997283
5 DecisionTreeClassifier 0.814 0.803797 0.671958 0.627096 0.186000 6.424290 0.716000 0.813810 0.735136
6 NuSVC 0.969 0.967334 0.936735 0.937996 0.020646 0.067079 0.578227 0.997543 0.997283
7 SGDClassifier 0.973 0.971307 0.944215 0.945813 0.018218 0.061125 0.576176 0.998270 0.998065
8 RandomForestClassifier 0.926 0.921610 0.854617 0.851564 0.076666 0.271507 0.744460 0.981208 0.978537
9 AdaBoostClassifier 0.921 0.916754 0.846304 0.841738 0.228336 0.649722 1.007489 0.979775 0.978109