Calibration

Functionality to calibrate a trained, binary classification model using temperature scaling.

---

source

ECELoss

def ECELoss(
    n_bins:int=10
):

Calculates the Expected Calibration Error of a model.

---

source

TemperatureSetter

def TemperatureSetter(
    model, lr:float=0.01, max_iter:int=1000, line_search_fn:NoneType=None, n_bins:int=10, verbose:bool=True
):

Calibrates a binary classification model optimizing temperature

---

source

ModelWithTemperature

def ModelWithTemperature(
    model
):

A decorator which wraps a model with temperature scaling

---

source

plot_calibration_curve

def plot_calibration_curve(
    labels, logits, cal_logits:NoneType=None, figsize:tuple=(6, 6), n_bins:int=10, strategy:str='uniform'
):

Call self as a function.

---

source

Learner.calibrate_model

def calibrate_model(
    X:NoneType=None, y:NoneType=None, lr:float=0.01, max_iter:int=10000, line_search_fn:NoneType=None, n_bins:int=10,
    strategy:str='uniform', show_plot:bool=True, figsize:tuple=(6, 6), verbose:bool=True
):

Call self as a function.

from tsai.basics import *

X, y, splits = get_UCR_data('FingerMovements', split_data=False)
tfms  = [None, TSClassification()]
batch_tfms = TSRobustScale()
# dls = get_ts_dls(X, y, splits=splits, tfms=tfms, batch_tfms=batch_tfms)
learn = TSClassifier(X, y, splits=splits, tfms=tfms, batch_tfms=batch_tfms,
                     arch='FCNPlus', metrics=accuracy)
learn.fit_one_cycle(2)

epoch	train_loss	valid_loss	accuracy	time
0	0.794469	0.731429	0.500000	00:00
1	0.758404	0.716087	0.490000	00:00

learn.calibrate_model()
calibrated_model = learn.calibrated_model

Before temperature - NLL: 0.716, ECE: 0.093
Calibrating the model...
...model calibrated
Optimal temperature: 272.026
After temperature  - NLL: 0.693, ECE: 0.010