nn-cnn-rnn-lstm-pytorch.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms","metadata":{"execution":{"iopub.status.busy":"2024-05-14T15:47:00.515767Z","iopub.execute_input":"2024-05-14T15:47:00.516175Z","iopub.status.idle":"2024-05-14T15:47:07.635333Z","shell.execute_reply.started":"2024-05-14T15:47:00.516144Z","shell.execute_reply":"2024-05-14T15:47:07.634117Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cude' if torch.cuda.is_available() else 'cpu')\ninput_size = 784 # 28x28 image size\n# --------- CNN -----------\nin_channels = 1 # only gray img\n# -------- RNN ---------\ninput_size_rnn = 28\nsequence_lenght = 28\nhidden_size = 256\nnum_layers = 2\n# ---------------------\nnum_classes = 10\nbatch_size = 64\nlearning_rate = 0.001\nnum_epochs = 2\n\nload_model = False","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:06:35.794184Z","iopub.execute_input":"2024-05-14T16:06:35.794833Z","iopub.status.idle":"2024-05-14T16:06:35.803308Z","shell.execute_reply.started":"2024-05-14T16:06:35.794800Z","shell.execute_reply":"2024-05-14T16:06:35.801690Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"class NN(nn.Module):\n    def __init__(self, input_size, num_classes):\n        super(NN,self).__init__()\n        self.fc1 = nn.Linear(input_size,50)\n        self.fc2 = nn.Linear(50,num_classes)\n        \n    def forward(self,x):\n        x = x.reshape(x.shape[0], -1)\n        x = F.relu(self.fc1(x))\n        x = self.fc2(x)\n        return x\n\nclass CNN(nn.Module):\n    def __init__(self, in_channels=1, num_classes = 10):\n        super(CNN, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=8, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n        self.pool = nn.MaxPool2d(kernel_size=(2,2), stride=(2,2))\n        self.conv2 = nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n        self.fc1 = nn.Linear(16*7*7, num_classes)\n        \n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n        x = self.pool(x)\n        x = F.relu(self.conv2(x))\n        x = self.pool(x)\n        x = x.reshape(x.shape[0], -1)\n        x = self.fc1(x)\n        \n        return x\n    \nclass RNN(nn.Module): # + LSTM\n    def __init__(self, input_size, hidden_size, num_layers, num_classes):\n        super(RNN, self).__init__()\n        self.hidden_size = hidden_size\n        self.num_layers = num_layers\n#         self.rnn = nn.RNN(input_size,hidden_size,num_layers, batch_first=True)\n        self.lstm = nn.LSTM(input_size,hidden_size,num_layers, batch_first=True)\n#         self.fc = nn.Linear(hidden_size*sequence_lenght, num_classes)\n        self.fc = nn.Linear(hidden_size, num_classes)\n        \n    def forward(self, x):\n        x = x.squeeze(1)\n        # x - torch.Size([64, 28, 28])\n        # h0 - torch.Size([2, 64, 256])\n        # out - torch.Size([64, 28, 256])\n        \n        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device) # hidden state - stores the data of prev step and may get flushed to zero through forget gate if irrelevent\n        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device) # cell state - stores data of whole network except of flused steps\n        # Forward Bias\n        out, _  = self.lstm(x,(h0,c0))\n        # out = out.reshape(out.shape[0], -1)\n        out = self.fc(out[:,-1,:])\n        \n        return out","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:05:42.838790Z","iopub.execute_input":"2024-05-14T16:05:42.839266Z","iopub.status.idle":"2024-05-14T16:05:42.858176Z","shell.execute_reply.started":"2024-05-14T16:05:42.839235Z","shell.execute_reply":"2024-05-14T16:05:42.856924Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"def save_checkpoint(state, filename='model_checkpoint.pth.tar'):\n    print('===> Saving Checkpoint')\n    torch.save(state, filename)\ndef load_checkpoint(checkpoint):\n    print('===> Loading Model')\n    model.load_state_dict(checkpoint['state_dict'])\n    optimizer.load_state_dict(checkpoint['optimizer'])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:05:45.507356Z","iopub.execute_input":"2024-05-14T16:05:45.507896Z","iopub.status.idle":"2024-05-14T16:05:45.515084Z","shell.execute_reply.started":"2024-05-14T16:05:45.507858Z","shell.execute_reply":"2024-05-14T16:05:45.513946Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data & Initializing Model","metadata":{}},{"cell_type":"code","source":"train_data = datasets.MNIST(root='/datasets', transform=transforms.ToTensor(), download=True, train=True)\ntrain_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True)\ntest_data = datasets.MNIST(root='/datasets', transform=transforms.ToTensor(), download=True, train=False)\ntest_loader = DataLoader(dataset=test_data, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:06:08.011245Z","iopub.execute_input":"2024-05-14T16:06:08.011857Z","iopub.status.idle":"2024-05-14T16:06:08.125143Z","shell.execute_reply.started":"2024-05-14T16:06:08.011731Z","shell.execute_reply":"2024-05-14T16:06:08.123545Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"# model = NN(input_size=input_size,num_classes=num_classes).to(device)\n# model = CNN(in_channels=in_channels,num_classes=num_classes).to(device)\nmodel = RNN(input_size_rnn, hidden_size, num_layers, num_classes).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:06:08.969914Z","iopub.execute_input":"2024-05-14T16:06:08.971405Z","iopub.status.idle":"2024-05-14T16:06:08.999246Z","shell.execute_reply.started":"2024-05-14T16:06:08.971351Z","shell.execute_reply":"2024-05-14T16:06:08.998248Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(),lr=learning_rate)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:06:10.957154Z","iopub.execute_input":"2024-05-14T16:06:10.957578Z","iopub.status.idle":"2024-05-14T16:06:10.963647Z","shell.execute_reply.started":"2024-05-14T16:06:10.957546Z","shell.execute_reply":"2024-05-14T16:06:10.962652Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"if load_model:\n    load_checkpoint(torch.load('model.pth.tar'))\n\nelse:\n    for epoch in range(num_epochs):\n        if epoch % 2 == 0 or epoch==(num_epochs-1):\n            checkpoint = {'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict()}\n            save_checkpoint(checkpoint)\n\n        for batch_idx, (data,targets) in enumerate(train_loader):\n            # get data to cuda if possible\n            data = data.to(device)\n            targets = targets.to(device)\n\n            # converting [64,1,28,28] to [64,784]\n            # data = data.reshape(data.shape[0], -1)\n\n            # forward\n            scores = model(data)\n            loss = criterion(scores, targets)\n\n            # backward\n            optimizer.zero_grad() # Nullyfing optimizers so it does save anything of back prop\n            loss.backward()\n\n            # gradient decent\n            optimizer.step()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:06:50.407519Z","iopub.execute_input":"2024-05-14T16:06:50.407922Z","iopub.status.idle":"2024-05-14T16:11:46.345324Z","shell.execute_reply.started":"2024-05-14T16:06:50.407894Z","shell.execute_reply":"2024-05-14T16:11:46.344233Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"===> Saving Checkpoint\n===> Saving Checkpoint\n","output_type":"stream"}]},{"cell_type":"code","source":"def get_accuracy(loader, model):\n    if loader.dataset.train:\n        print('Checking accuracy on training data')\n    else:\n        print('Checking accuracy on testing data')\n        \n    num_currect = 0\n    num_samples = 0\n    model.eval()\n    \n    with torch.no_grad():\n        for x,y in loader:\n            x = x.to(device)\n            y = y.to(device)\n            \n            scores = model(x)\n            _, predictions = scores.max(1)\n            num_currect += (predictions == y).sum()\n            num_samples += predictions.size(0)\n            \n        print(f'got {num_currect} / {num_samples} samples with accuracy: {float(num_currect)/float(num_samples)*100:.2f}')\n        \n    model.train()","metadata":{"execution":{"iopub.status.busy":"2024-05-12T17:08:42.647827Z","iopub.execute_input":"2024-05-12T17:08:42.648741Z","iopub.status.idle":"2024-05-12T17:08:42.657103Z","shell.execute_reply.started":"2024-05-12T17:08:42.648700Z","shell.execute_reply":"2024-05-12T17:08:42.656016Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"get_accuracy(train_loader,model)\nget_accuracy(test_loader,model)","metadata":{"execution":{"iopub.status.busy":"2024-05-12T17:08:42.658519Z","iopub.execute_input":"2024-05-12T17:08:42.658930Z","iopub.status.idle":"2024-05-12T17:09:33.341201Z","shell.execute_reply.started":"2024-05-12T17:08:42.658894Z","shell.execute_reply":"2024-05-12T17:09:33.340279Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stdout","text":"Checking accuracy on training data\ngot 58591 / 60000 samples with accuracy: 97.65\nChecking accuracy on testing data\ngot 9757 / 10000 samples with accuracy: 97.57\n","output_type":"stream"}]},{"cell_type":"markdown","source":"read this: https://d2l.ai/chapter_recurrent-modern/lstm.html","metadata":{}}]}