update doc. add yaml configs for tuto; close #30; close #4;

Author: slegroux

.docker/build.sh

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+
+docker build --network host -t nimrod-dev -f Dockerfile .

.gitignore

@@ -141,7 +141,7 @@ tests/
 *tfevents*
 */wandb/*
 *_logs*
-recipes/image/
+# recipes/image/
 logs/
 _proc/
 _docs/

.vscode/settings.json

@@ -0,0 +1,2 @@
+{
+}

README.md

@@ -24,10 +24,20 @@ pip install nimrod
 Check recipes in `recipes/` folder. For instance:
 
 ``` bash
-cd recipes/autoencoder/
-python train.py
+cd recipes/images/mnist
+python train.py datamodule.num_workers=10 trainer.max_epochs=20 trainer.accelerator='gpu'
+head conf/train.yaml
 ```
 
+All the parameters of the experiment are editable and read from a .yaml
+file which details:
+
+- data and logging directory paths
+- data module with data source path and batching parameters
+- model architecture
+- trainer with hardware acceleration and number of epochs
+- callbacks for early stopping and automatic logging to Wandb
+
 ## Docker
 
 You might want to use docker containers for reproductible development

config/data/image/mnist.yaml

@@ -1,7 +1,14 @@
+dataset:
+  _target_: nimrod.image.datasets.MNISTDataset
+  data_dir: "../data/image"
+  train: False
+  transform: 
+    _target_: torchvision.transforms.ToTensor
+
 datamodule:
   _target_: nimrod.image.datasets.MNISTDataModule
   data_dir: "../data/image"
   train_val_test_split: [0.8, 0.1, 0.1]
-  batch_size: 1024
+  batch_size: 64
   num_workers: 0
   pin_memory: False

nimrod/_modidx.py

@@ -357,6 +357,9 @@
                                                                                                         'nimrod/models/autoencoders.py'),
                                             'nimrod.models.autoencoders.AutoEncoderPL.validation_step': ( 'models.autoencoders.html#autoencoderpl.validation_step',
                                                                                                           'nimrod/models/autoencoders.py')},
+            'nimrod.models.conv': { 'nimrod.models.conv.ConvNet': ('models.conv.html#convnet', 'nimrod/models/conv.py'),
+                                    'nimrod.models.conv.ConvNet.__init__': ('models.conv.html#convnet.__init__', 'nimrod/models/conv.py'),
+                                    'nimrod.models.conv.ConvNet.forward': ('models.conv.html#convnet.forward', 'nimrod/models/conv.py')},
             'nimrod.models.lm': { 'nimrod.models.lm.NNLM': ('models.lm.html#nnlm', 'nimrod/models/lm.py'),
                                   'nimrod.models.lm.NNLM.__init__': ('models.lm.html#nnlm.__init__', 'nimrod/models/lm.py'),
                                   'nimrod.models.lm.NNLM.forward': ('models.lm.html#nnlm.forward', 'nimrod/models/lm.py'),
@@ -786,4 +789,5 @@
                                                                                              'nimrod/tts/models/valle.py'),
                                          'nimrod.tts.models.valle.TokenEmbedding.weight': ( 'tts.models.valle.html#tokenembedding.weight',
                                                                                             'nimrod/tts/models/valle.py')},
-            'nimrod.utils': {'nimrod.utils.get_device': ('utils.html#get_device', 'nimrod/utils.py')}}}
+            'nimrod.utils': { 'nimrod.utils.get_device': ('utils.html#get_device', 'nimrod/utils.py'),
+                              'nimrod.utils.set_seed': ('utils.html#set_seed', 'nimrod/utils.py')}}}

nimrod/image/datasets.py

@@ -29,14 +29,19 @@ class ImageDataset(Dataset):
     def show_idx(self,
         index:int # Index of the (image,label) sample to visualize
         ):
+        "display image from data point index of a image dataset"
         X, y = self.__getitem__(index)
         plt.figure(figsize = (1, 1))
         plt.imshow(X.numpy().reshape(28,28),cmap='gray')
         plt.title(f"Label: {int(y)}")
         plt.show()
 
     @staticmethod
-    def show_grid(imgs, save_path=None):
+    def show_grid(
+        imgs: List[torch.Tensor], # python list of images dim (C,H,W)
+        save_path=None # path where image can be saved
+        ):
+        "display list of mnist-like images (C,H,W)"
         if not isinstance(imgs, list):
             imgs = [imgs]
         fig, axs = plt.subplots(ncols=len(imgs), squeeze=False)
@@ -50,6 +55,7 @@ def show_grid(imgs, save_path=None):
     def show_random(self,
         n=3 # number of images to display
         ):
+        "display grid of random images"
         indices = torch.randint(0,len(self), (n,))
         images = []
         for index in indices:
@@ -59,7 +65,7 @@ def show_random(self,
         self.show_grid(images)
 
 
-# %% ../../nbs/image.datasets.ipynb 8
+# %% ../../nbs/image.datasets.ipynb 11
 class MNISTDataset(ImageDataset):
     "MNIST digit dataset"
 
@@ -68,6 +74,9 @@ def __init__(
         data_dir:str='~/Data', # path where data is saved
         train = True, # train or test dataset
         transform:torchvision.transforms.transforms=torchvision.transforms.ToTensor() # data formatting
+        # TODO: add noramlization?
+        # torchvision.transforms.Compose([torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(0.1307,), (0.3081,))])
+
     ):
 
         super().__init__()
@@ -79,18 +88,19 @@ def __init__(
             download=True
         )
 
-    def __len__(self):
+    def __len__(self) -> int: # length of dataset
         return len(self.ds)
 
-    def __getitem__(self, idx):
+    def __getitem__(self, idx # index into the dataset
+                    ) -> tuple[torch.FloatTensor, int]: # Y image data, x digit number
         x = self.ds[idx][0]
         y = self.ds[idx][1]
         return x, y
 
     def train_dev_split(self,
         ratio:float, # percentage of train/dev split,
         seed:int=42 # rand generator seed
-    ):
+    ) -> tuple[torchvision.datasets.MNIST, torchvision.datasets.MNIST]: # train and set mnnist datasets
         train_set_size = int(len(self.ds) * ratio)
         valid_set_size = len(self.ds) - train_set_size
 
@@ -101,15 +111,15 @@ def train_dev_split(self,
 
 
 
-# %% ../../nbs/image.datasets.ipynb 14
+# %% ../../nbs/image.datasets.ipynb 18
 class MNISTDataModule(LightningDataModule):
     def __init__(
         self,
-        data_dir: str = "~/Data/",
-        train_val_test_split:List[float] = [0.8, 0.1, 0.1],
-        batch_size: int = 64,
-        num_workers: int = 0,
-        pin_memory: bool = False,
+        data_dir: str = "~/Data/", # path to source data dir
+        train_val_test_split:List[float] = [0.8, 0.1, 0.1], # train val test %
+        batch_size: int = 64, # size of compute batch
+        num_workers: int = 0, # num_workers equal 0 means that it’s the main process that will do the data loading when needed, num_workers equal 1 is the same as any n, but you’ll only have a single worker, so it might be slow
+        pin_memory: bool = False, # If you load your samples in the Dataset on CPU and would like to push it during training to the GPU, you can speed up the host to device transfer by enabling pin_memory. This lets your DataLoader allocate the samples in page-locked memory, which speeds-up the transfer
     ):
         super().__init__()
         self.save_hyperparameters(logger=False) # can access inputs with self.hparams
@@ -122,17 +132,19 @@ def __init__(
             raise Exception('split percentages should sum up to 1.0')
 
     @property
-    def num_classes(self):
+    def num_classes(self) -> int: # num of classes in dataset
         return 10
 
-    def prepare_data(self):
+    def prepare_data(self) -> None:
         """Download data if needed + format with MNISTDataset
         """
         MNISTDataset(self.hparams.data_dir, train=True)
         MNISTDataset(self.hparams.data_dir, train=False)
 
-    def setup(self, stage: Optional[str] = None):
+    def setup(self, stage: Optional[str] = None) -> None:
+        # concat train & test mnist dataset and randomly generate train, eval, test sets
         if not self.data_train and not self.data_val and not self.data_test:
+            # ((B, H, W), int)
             trainset = MNISTDataset(self.hparams.data_dir, train=True, transform=self.transforms)
             testset = MNISTDataset(self.hparams.data_dir, train=False, transform=self.transforms)
             dataset = ConcatDataset(datasets=[trainset, testset])
@@ -143,7 +155,7 @@ def setup(self, stage: Optional[str] = None):
                 generator=torch.Generator().manual_seed(42),
             )
 
-    def train_dataloader(self):
+    def train_dataloader(self) -> torch.utils.data.DataLoader:
         return DataLoader(
             dataset=self.data_train,
             batch_size=self.hparams.batch_size,
@@ -152,7 +164,7 @@ def train_dataloader(self):
             shuffle=True,
         )
 
-    def val_dataloader(self):
+    def val_dataloader(self) -> torch.utils.data.DataLoader:
         return DataLoader(
             dataset=self.data_val,
             batch_size=self.hparams.batch_size,
@@ -161,7 +173,7 @@ def val_dataloader(self):
             shuffle=False,
         )
 
-    def test_dataloader(self):
+    def test_dataloader(self) -> torch.utils.data.DataLoader:
         return DataLoader(
             dataset=self.data_test,
             batch_size=self.hparams.batch_size,
@@ -170,7 +182,7 @@ def test_dataloader(self):
             shuffle=False,
         )
 
-    def teardown(self, stage: Optional[str] = None):
+    def teardown(self, stage: Optional[str] = None) -> None:
         """Clean up after fit or test."""
         pass
 

nimrod/models/conv.py

@@ -0,0 +1,57 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/models.conv.ipynb.
+
+# %% auto 0
+__all__ = ['ConvNet']
+
+# %% ../../nbs/models.conv.ipynb 3
+import torch.nn as nn
+import torch
+from torchvision.transforms import ToTensor
+from torch.utils.data import DataLoader
+from torchvision.datasets import MNIST
+
+from pytorch_lightning import LightningModule, Trainer
+from torchmetrics import Accuracy
+from hydra.utils import instantiate
+from omegaconf import OmegaConf
+
+from ..data.datasets import MNISTDataModule
+from ..utils import get_device
+
+# %% ../../nbs/models.conv.ipynb 5
+class ConvNet(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+        # Define the convolutional layers
+        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, padding=1)
+        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, padding=1)
+
+        # Define the pooling and dropout layers
+        self.pool = nn.MaxPool2d(2, 2)
+        self.dropout1 = nn.Dropout(0.25)
+        self.dropout2 = nn.Dropout(0.5)
+
+        # Define the fully connected layers
+        self.fc1 = nn.Linear(32 * 7 * 7, 128)
+        self.fc2 = nn.Linear(128, 10)
+
+    def forward(self, x:torch.Tensor # input image tensor of dimension (B, C, W, H)
+                ) -> torch.Tensor: # output probs (B, N_classes)
+        # Pass the input through the convolutional layers
+        x = self.conv1(x)
+        x = self.pool(x)
+        x = self.dropout1(x)
+        x = self.conv2(x)
+        x = self.pool(x)
+        x = self.dropout2(x)
+
+        # Reshape the output for the fully connected layers
+        x = x.view(-1, 32 * 7 * 7)
+
+        # Pass the output through the fully connected layers
+        x = self.fc1(x)
+        x = self.fc2(x)
+
+        # Return the final output
+        return x

nimrod/models/mlp.py

@@ -3,34 +3,49 @@
 # %% auto 0
 __all__ = ['MLP', 'MLP_PL']
 
-# %% ../../nbs/models.mlp.ipynb 3
+# %% ../../nbs/models.mlp.ipynb 4
 import torch.nn as nn
 import torch
+from torchvision.transforms import ToTensor
+from torch.utils.data import DataLoader
+from torchvision.datasets import MNIST
 
-from pytorch_lightning import LightningModule
+
+from pytorch_lightning import LightningModule, Trainer
 from torchmetrics import Accuracy
 from hydra.utils import instantiate
 from omegaconf import OmegaConf
 
 from ..data.datasets import MNISTDataModule
+from ..utils import get_device
+
+# from IPython.core.debugger import set_trace
 
-# %% ../../nbs/models.mlp.ipynb 5
+# %% ../../nbs/models.mlp.ipynb 6
 class MLP(nn.Module):
-    def __init__(self, n_in=32*32*3, n_h=64, n_out=10):
+    def __init__(
+                self, n_in:int=32*32*3, # input dimension e.g. (H,W) for image
+                n_h:int=64, # hidden dimension
+                n_out:int=10 # output dimension (= number of classes for classification)
+                ):
         super().__init__()
         l1 = nn.Linear(n_in, n_h)
         l2 = nn.Linear(n_h, n_out)
-        relu = nn.ReLU()
-        self.layers = nn.Sequential(l1,l2,relu)
+        dropout = nn.Dropout(0.2)
+        self.layers = nn.Sequential(l1,l2, dropout)
 
-    def forward(self, x):
+    def forward(self, x: torch.FloatTensor # dim (B, H*W)
+                ) -> torch.FloatTensor:
         return self.layers(x)
 
-# %% ../../nbs/models.mlp.ipynb 9
+# %% ../../nbs/models.mlp.ipynb 20
 class MLP_PL(LightningModule):
-    def __init__(self, mlp:MLP):
+    def __init__(self,
+                mlp:MLP # pure pytorch MLP model
+                ):
         super().__init__()
-        self.save_hyperparameters(ignore=['mlp'])
+        # self.save_hyperparameters(ignore=['mlp'])
+        self.save_hyperparameters()
         self.mlp = mlp
         self.loss = nn.CrossEntropyLoss()
         self.accuracy = Accuracy(task="multiclass", num_classes=10)
@@ -39,7 +54,9 @@ def configure_optimizers(self):
         optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
         return optimizer
 
-    def forward(self, x):
+    def forward(self,
+                x: torch.Tensor # X input images dim(B, H*W)
+                ) -> torch.Tensor: # y class probabilities (B, n_classes)
         return(self.mlp(x))
 
     def training_step(self, batch, batch_idx):

nimrod/utils.py

@@ -1,10 +1,13 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/utils.ipynb.
 
 # %% auto 0
-__all__ = ['get_device']
+__all__ = ['get_device', 'set_seed']
 
 # %% ../nbs/utils.ipynb 4
 import torch
+import numpy as np
+import random
+import os
 
 # %% ../nbs/utils.ipynb 5
 def get_device():
@@ -13,3 +16,17 @@ def get_device():
     else:
         device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     return device
+
+# %% ../nbs/utils.ipynb 9
+def set_seed(seed: int = 42) -> None:
+    np.random.seed(seed)
+    random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    # When running on the CuDNN backend, two further options must be set
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    # Set a fixed value for the hash seed
+    os.environ["PYTHONHASHSEED"] = str(seed)
+    print(f"Random seed set as {seed}")
+