0% accuracy on test set ?

[kevinPoliPoli]

Discussed in #895

^{Originally posted by kevinPoliPoli April 21, 2024}
Hi everyone, i recently trainer a ConvTranPlus (default parameters) model for a multi class problem (21 classes, 4 features) obtaining 94% of accuracy on the validation set (image below)

(sorry for the image, in the y axis loss goes from 3.0 to 0.5)

As we can see the valid loss is below the training loss level, so the model has an high generalization capability...
The problem is that when I try to make predictions on the same training set I used to train the model I obtain 0-4% accuracy. How it is possible? I also tried both with train and valid set, obtaining always 0-4 % accuracy.
Moreover, making inference on the same set multiple times, I obtain different results (0-4%)

ps: the dataset is private, I can't provide it here
Thank you in advance :)

Here the code I wrote:

relabeling

  class_map = {
      0.8:80, 0.81:81, 0.82:82, 0.83:83, 0.84:84, 0.85:85, 0.86:86,
      0.87:87, 0.88:88, 0.89:89, 0.9:90, 0.91:91, 0.92:92, 0.93:93,
      0.94:94, 0.95:95, 0.96:96, 0.97:97, 0.98:98, 0.99:99, 1.0:100,
  }

create dataset and splits

window_length = 60  
stride = 30 # n datapoints the window is moved ahead along the sequence. default = 1
validation_size = .20

# obtain dataset and reduce it if needed
df = pd.read_parquet('./datasets/old/resampled/ds_sim_resampled.parquet')
print(df.shape)

blocks = []
end_index = len(df) - window_length + 1
for i in range(0, end_index, stride):
    window = df.iloc[i:i+window_length] 
    
    unique_labels = window['SOH'].unique()
    
    if len(unique_labels) == 1:
        blocks.append(window)
consistent_df = pd.concat(blocks)
print(consistent_df.shape)

# convert dataframe into TSDataset compatible input
columns = ["Voltage", "Current", "SOC", "Temperature"]
X, y = SlidingWindow(window_length, get_x=columns, get_y='SOH', stride=stride)(consistent_df)
del(df)
print(f"Shapes - X: {X.shape} - y: {y.shape}")

# apply relabeling
labeler = ReLabeler(class_map)
y = labeler(y)

# define splits based on validation size
splits_train = get_splits(y, valid_size=validation_size, balance=True, random_state=23, shuffle=True)

apply standardization and prepare dataloader

# compute mean and std from X: save it to scale the test dataset later
mean = np.mean(X, axis=(0, 2))
std = np.std(X, axis=(0, 2))
print("old mean:", mean)  # shape (4,)
print("old std:", std)  # shape (4,)
mean_expanded = np.expand_dims(mean, axis=(0, 2))
std_expanded = np.expand_dims(std, axis=(0, 2))

# apply standardization to X 
X_standardized = (X - mean_expanded) / std_expanded
mean = np.mean(X_standardized, axis=(0, 2))
std = np.std(X_standardized, axis=(0, 2))
print("new mean:", mean)
print("new std:", std) 

bs_t = 64  # train batch size
bs_v = 64  # validation batch size

ds_tfms  = [None, [TSClassification()]]  # dataset transformations
dl_tfms = [None]

dsets = TSDatasets(X_standardized, y, tfms=ds_tfms, splits=splits_train, inplace=False)
dls   = TSDataLoaders.from_dsets(dsets.train, dsets.valid, bs=[bs_t, bs_v], batch_tfms=dl_tfms, shuffle=True, num_workers=4).cuda()

build learner to find LR

arch = "ConvTranPlus"

metrics = [RocAucBinary(multi_class='ovo'), accuracy]
loss_func = CrossEntropyLossFlat()
opt_func = partial(SGD, mom=0.9)

model = ConvTranPlus(dls.vars, dls.c, dls.len)

learn = Learner(dls, model, loss_func=loss_func, metrics=metrics,cbs=ShowGraphCallback2())
lr_max = learn.lr_find().valley
print(lr_max)

train

from fastai.callback.tracker import EarlyStoppingCallback
from fastai.callback.schedule import *

# hyper-parameters
lr = 0.0006309573538601399  # found by some tests
n_epochs = 250
use_wandb = False
cbs = [EarlyStoppingCallback(min_delta=0.001, patience=3), ShowGraphCallback2(), PredictionDynamics(figsize=(6,5))] 

metrics = [accuracy, Precision(average='macro'), Recall(average='macro'), F1Score(average='macro'), RocAuc()]
loss_func = CrossEntropyLossFlat()

model = ConvTranPlus(dls.vars, dls.c, dls.len)
learn = Learner(dls, model, loss_func=loss_func, metrics=metrics, cbs=ShowGraphCallback2())

learn.fit_one_cycle(n_epochs, lr)

save the model

# metadata 
arch = "ConvTranPlus"
realorsim = 'simold'
filename = f'{arch}-n_epochs{n_epochs}-bs_t{bs_t}-bs_v{bs_v}-lr{lr}-type{realorsim}-norm-acc94'
PATH = Path(f'./models/{arch}/inference/{filename}.pkl')
learn.export(PATH)  # for inference only
PATH = Path(f'./{arch}/withstate/{filename}-state')  # autosave into ./models.
learn.save(PATH)  # save model and optimizer state -> used for finetuning or incremental learning

make inference

from fastai.learner import load_learner
from tsai.inference import get_X_preds
from sklearn.metrics import accuracy_score
import numpy as np

filename = "correctModelName"
PATH = Path(f'./models/{arch}/inference/{filename}.pkl')
inference_learn = load_learner(PATH)  # import model 

probs, target, predictions = inference_learn.get_X_preds(X_standardized[splits_train[0]])
probs = probs.cpu().numpy()
#target = target.cpu().numpy()
predictions_int = list(map(int, predictions))

print("predictions", np.array(predictions_int))

accuracy = accuracy_score(y[splits_train[0]], predictions_int)
print("Accuracy:", accuracy)
```</div>