Skip to content

Commit b1266f0

Browse files
slegrouxslegroux
committed
fix resblock
1 parent 7c51c34 commit b1266f0

File tree

6 files changed

+602
-187
lines changed

6 files changed

+602
-187
lines changed

‎nbs/models.conv.ipynb

+114-76
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,16 @@
3232
"cell_type": "code",
3333
"execution_count": null,
3434
"metadata": {},
35-
"outputs": [],
35+
"outputs": [
36+
{
37+
"name": "stderr",
38+
"output_type": "stream",
39+
"text": [
40+
"[23:23:58] INFO - PyTorch version 2.5.1 available.\n",
41+
"Seed set to 42\n"
42+
]
43+
}
44+
],
3645
"source": [
3746
"#| export\n",
3847
"import torch.nn as nn\n",
@@ -96,11 +105,11 @@
96105
"name": "stderr",
97106
"output_type": "stream",
98107
"text": [
99-
"[19:18:28] INFO - Init ImageDataModule for mnist\n",
100-
"[19:18:32] INFO - loading dataset mnist with args () from split train\n",
101-
"[19:18:40] INFO - loading dataset mnist with args () from split test\n",
102-
"[19:18:42] INFO - split train into train/val [0.8, 0.2]\n",
103-
"[19:18:43] INFO - train: 48000 val: 12000, test: 10000\n"
108+
"[23:24:00] INFO - Init ImageDataModule for mnist\n",
109+
"[23:24:05] INFO - loading dataset mnist with args () from split train\n",
110+
"[23:24:12] INFO - loading dataset mnist with args () from split test\n",
111+
"[23:24:15] INFO - split train into train/val [0.8, 0.2]\n",
112+
"[23:24:15] INFO - train: 48000 val: 12000, test: 10000\n"
104113
]
105114
}
106115
],
@@ -325,7 +334,7 @@
325334
"text/markdown": [
326335
"---\n",
327336
"\n",
328-
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L37){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
337+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L38){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
329338
"\n",
330339
"### ConvLayer\n",
331340
"\n",
@@ -380,7 +389,7 @@
380389
"text/plain": [
381390
"---\n",
382391
"\n",
383-
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L37){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
392+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L38){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
384393
"\n",
385394
"### ConvLayer\n",
386395
"\n",
@@ -452,7 +461,7 @@
452461
"text/markdown": [
453462
"---\n",
454463
"\n",
455-
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L75){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
464+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L77){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
456465
"\n",
457466
"### ConvLayer.forward\n",
458467
"\n",
@@ -468,7 +477,7 @@
468477
"text/plain": [
469478
"---\n",
470479
"\n",
471-
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L75){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
480+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L77){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
472481
"\n",
473482
"### ConvLayer.forward\n",
474483
"\n",
@@ -507,7 +516,7 @@
507516
"name": "stderr",
508517
"output_type": "stream",
509518
"text": [
510-
"[19:18:43] WARNING - setting conv bias back to False as Batchnorm is used\n"
519+
"[23:24:16] WARNING - setting conv bias back to False as Batchnorm is used\n"
511520
]
512521
},
513522
{
@@ -614,7 +623,7 @@
614623
"output_type": "stream",
615624
"text": [
616625
"Seed set to 42\n",
617-
"[19:18:43] WARNING - setting conv bias back to False as Batchnorm is used\n"
626+
"[23:24:16] WARNING - setting conv bias back to False as Batchnorm is used\n"
618627
]
619628
},
620629
{
@@ -706,7 +715,7 @@
706715
"text/markdown": [
707716
"---\n",
708717
"\n",
709-
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L37){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
718+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L38){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
710719
"\n",
711720
"### ConvLayer\n",
712721
"\n",
@@ -761,7 +770,7 @@
761770
"text/plain": [
762771
"---\n",
763772
"\n",
764-
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L37){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
773+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L38){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
765774
"\n",
766775
"### ConvLayer\n",
767776
"\n",
@@ -1048,60 +1057,112 @@
10481057
"text/markdown": [
10491058
"---\n",
10501059
"\n",
1051-
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L37){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1060+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L38){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
10521061
"\n",
10531062
"### ConvLayer\n",
10541063
"\n",
10551064
"> ConvLayer (in_channels:int=3, out_channels:int=16, kernel_size:int=3,\n",
1056-
"> stride:int=2, bias:bool=True, normalization:Optional[Type[torc\n",
1057-
"> h.nn.modules.module.Module]]=<class\n",
1065+
"> stride:int=2, bias:bool=False, normalization:Optional[Type[tor\n",
1066+
"> ch.nn.modules.module.Module]]=<class\n",
10581067
"> 'torch.nn.modules.batchnorm.BatchNorm2d'>, activation:Optional\n",
10591068
"> [Type[torch.nn.modules.module.Module]]=<class\n",
10601069
"> 'torch.nn.modules.activation.ReLU'>)\n",
10611070
"\n",
1062-
"*A 2D convolutional layer with optional batch normalization and activation.\n",
1071+
"*Base class for all neural network modules.\n",
1072+
"\n",
1073+
"Your models should also subclass this class.\n",
1074+
"\n",
1075+
"Modules can also contain other Modules, allowing to nest them in\n",
1076+
"a tree structure. You can assign the submodules as regular attributes::\n",
10631077
"\n",
1064-
"This layer performs 2D convolution with stride 2 for downsampling, optionally followed by\n",
1065-
"batch normalization and activation.*\n",
1078+
" import torch.nn as nn\n",
1079+
" import torch.nn.functional as F\n",
1080+
"\n",
1081+
" class Model(nn.Module):\n",
1082+
" def __init__(self) -> None:\n",
1083+
" super().__init__()\n",
1084+
" self.conv1 = nn.Conv2d(1, 20, 5)\n",
1085+
" self.conv2 = nn.Conv2d(20, 20, 5)\n",
1086+
"\n",
1087+
" def forward(self, x):\n",
1088+
" x = F.relu(self.conv1(x))\n",
1089+
" return F.relu(self.conv2(x))\n",
1090+
"\n",
1091+
"Submodules assigned in this way will be registered, and will have their\n",
1092+
"parameters converted too when you call :meth:`to`, etc.\n",
1093+
"\n",
1094+
".. note::\n",
1095+
" As per the example above, an ``__init__()`` call to the parent class\n",
1096+
" must be made before assignment on the child.\n",
1097+
"\n",
1098+
":ivar training: Boolean represents whether this module is in training or\n",
1099+
" evaluation mode.\n",
1100+
":vartype training: bool*\n",
10661101
"\n",
10671102
"| | **Type** | **Default** | **Details** |\n",
10681103
"| -- | -------- | ----------- | ----------- |\n",
10691104
"| in_channels | int | 3 | input channels |\n",
10701105
"| out_channels | int | 16 | output channels |\n",
10711106
"| kernel_size | int | 3 | kernel size |\n",
10721107
"| stride | int | 2 | stride |\n",
1073-
"| bias | bool | True | If True, adds a learnable bias to the convolution |\n",
1074-
"| normalization | Optional | BatchNorm2d | Normalization layer to use after convolution |\n",
1075-
"| activation | Optional | ReLU | Activation function to use after normalization |"
1108+
"| bias | bool | False | |\n",
1109+
"| normalization | Optional | BatchNorm2d | |\n",
1110+
"| activation | Optional | ReLU | |"
10761111
],
10771112
"text/plain": [
10781113
"---\n",
10791114
"\n",
1080-
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L37){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
1115+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/models/conv.py#L38){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
10811116
"\n",
10821117
"### ConvLayer\n",
10831118
"\n",
10841119
"> ConvLayer (in_channels:int=3, out_channels:int=16, kernel_size:int=3,\n",
1085-
"> stride:int=2, bias:bool=True, normalization:Optional[Type[torc\n",
1086-
"> h.nn.modules.module.Module]]=<class\n",
1120+
"> stride:int=2, bias:bool=False, normalization:Optional[Type[tor\n",
1121+
"> ch.nn.modules.module.Module]]=<class\n",
10871122
"> 'torch.nn.modules.batchnorm.BatchNorm2d'>, activation:Optional\n",
10881123
"> [Type[torch.nn.modules.module.Module]]=<class\n",
10891124
"> 'torch.nn.modules.activation.ReLU'>)\n",
10901125
"\n",
1091-
"*A 2D convolutional layer with optional batch normalization and activation.\n",
1126+
"*Base class for all neural network modules.\n",
1127+
"\n",
1128+
"Your models should also subclass this class.\n",
1129+
"\n",
1130+
"Modules can also contain other Modules, allowing to nest them in\n",
1131+
"a tree structure. You can assign the submodules as regular attributes::\n",
10921132
"\n",
1093-
"This layer performs 2D convolution with stride 2 for downsampling, optionally followed by\n",
1094-
"batch normalization and activation.*\n",
1133+
" import torch.nn as nn\n",
1134+
" import torch.nn.functional as F\n",
1135+
"\n",
1136+
" class Model(nn.Module):\n",
1137+
" def __init__(self) -> None:\n",
1138+
" super().__init__()\n",
1139+
" self.conv1 = nn.Conv2d(1, 20, 5)\n",
1140+
" self.conv2 = nn.Conv2d(20, 20, 5)\n",
1141+
"\n",
1142+
" def forward(self, x):\n",
1143+
" x = F.relu(self.conv1(x))\n",
1144+
" return F.relu(self.conv2(x))\n",
1145+
"\n",
1146+
"Submodules assigned in this way will be registered, and will have their\n",
1147+
"parameters converted too when you call :meth:`to`, etc.\n",
1148+
"\n",
1149+
".. note::\n",
1150+
" As per the example above, an ``__init__()`` call to the parent class\n",
1151+
" must be made before assignment on the child.\n",
1152+
"\n",
1153+
":ivar training: Boolean represents whether this module is in training or\n",
1154+
" evaluation mode.\n",
1155+
":vartype training: bool*\n",
10951156
"\n",
10961157
"| | **Type** | **Default** | **Details** |\n",
10971158
"| -- | -------- | ----------- | ----------- |\n",
10981159
"| in_channels | int | 3 | input channels |\n",
10991160
"| out_channels | int | 16 | output channels |\n",
11001161
"| kernel_size | int | 3 | kernel size |\n",
11011162
"| stride | int | 2 | stride |\n",
1102-
"| bias | bool | True | If True, adds a learnable bias to the convolution |\n",
1103-
"| normalization | Optional | BatchNorm2d | Normalization layer to use after convolution |\n",
1104-
"| activation | Optional | ReLU | Activation function to use after normalization |"
1163+
"| bias | bool | False | |\n",
1164+
"| normalization | Optional | BatchNorm2d | |\n",
1165+
"| activation | Optional | ReLU | |"
11051166
]
11061167
},
11071168
"execution_count": null,
@@ -1135,47 +1196,24 @@
11351196
"==========================================================================================\n",
11361197
"ConvNet [64, 10] --\n",
11371198
"├─Sequential: 1-1 [64, 10] --\n",
1138-
"│ └─ConvLayer: 2-1 [64, 8, 28, 28] --\n",
1139-
"│ │ └─Sequential: 3-1 [64, 8, 28, 28] --\n",
1140-
"│ │ │ └─Conv2d: 4-1 [64, 8, 28, 28] 72\n",
1141-
"│ │ │ └─BatchNorm2d: 4-2 [64, 8, 28, 28] 16\n",
1142-
"│ │ │ └─ReLU: 4-3 [64, 8, 28, 28] --\n",
1143-
"│ └─ConvLayer: 2-2 [64, 16, 14, 14] --\n",
1144-
"│ │ └─Sequential: 3-2 [64, 16, 14, 14] --\n",
1145-
"│ │ │ └─Conv2d: 4-4 [64, 16, 14, 14] 1,152\n",
1146-
"│ │ │ └─BatchNorm2d: 4-5 [64, 16, 14, 14] 32\n",
1147-
"│ │ │ └─ReLU: 4-6 [64, 16, 14, 14] --\n",
1148-
"│ └─ConvLayer: 2-3 [64, 32, 7, 7] --\n",
1149-
"│ │ └─Sequential: 3-3 [64, 32, 7, 7] --\n",
1150-
"│ │ │ └─Conv2d: 4-7 [64, 32, 7, 7] 4,608\n",
1151-
"│ │ │ └─BatchNorm2d: 4-8 [64, 32, 7, 7] 64\n",
1152-
"│ │ │ └─ReLU: 4-9 [64, 32, 7, 7] --\n",
1153-
"│ └─ConvLayer: 2-4 [64, 64, 4, 4] --\n",
1154-
"│ │ └─Sequential: 3-4 [64, 64, 4, 4] --\n",
1155-
"│ │ │ └─Conv2d: 4-10 [64, 64, 4, 4] 18,432\n",
1156-
"│ │ │ └─BatchNorm2d: 4-11 [64, 64, 4, 4] 128\n",
1157-
"│ │ │ └─ReLU: 4-12 [64, 64, 4, 4] --\n",
1158-
"│ └─ConvLayer: 2-5 [64, 128, 2, 2] --\n",
1159-
"│ │ └─Sequential: 3-5 [64, 128, 2, 2] --\n",
1160-
"│ │ │ └─Conv2d: 4-13 [64, 128, 2, 2] 73,728\n",
1161-
"│ │ │ └─BatchNorm2d: 4-14 [64, 128, 2, 2] 256\n",
1162-
"│ │ │ └─ReLU: 4-15 [64, 128, 2, 2] --\n",
1163-
"│ └─ConvLayer: 2-6 [64, 10, 1, 1] --\n",
1164-
"│ │ └─Sequential: 3-6 [64, 10, 1, 1] --\n",
1165-
"│ │ │ └─Conv2d: 4-16 [64, 10, 1, 1] 11,520\n",
1166-
"│ │ │ └─BatchNorm2d: 4-17 [64, 10, 1, 1] 20\n",
1167-
"│ │ │ └─ReLU: 4-18 [64, 10, 1, 1] --\n",
1168-
"│ └─Flatten: 2-7 [64, 10] --\n",
1199+
"│ └─ConvLayer: 2-1 [64, 8, 28, 28] 88\n",
1200+
"│ └─ConvLayer: 2-2 [64, 16, 14, 14] 1,184\n",
1201+
"│ └─ConvLayer: 2-3 [64, 32, 7, 7] 4,672\n",
1202+
"│ └─ConvLayer: 2-4 [64, 64, 4, 4] 18,560\n",
1203+
"│ └─ConvLayer: 2-5 [64, 128, 2, 2] 73,984\n",
1204+
"│ └─ConvLayer: 2-6 [64, 64, 1, 1] 73,856\n",
1205+
"│ └─ConvLayer: 2-7 [64, 10, 1, 1] 5,780\n",
1206+
"│ └─Flatten: 2-8 [64, 10] --\n",
11691207
"==========================================================================================\n",
1170-
"Total params: 110,028\n",
1171-
"Trainable params: 110,028\n",
1208+
"Total params: 178,124\n",
1209+
"Trainable params: 178,124\n",
11721210
"Non-trainable params: 0\n",
1173-
"Total mult-adds (Units.MEGABYTES): 71.03\n",
1211+
"Total mult-adds (Units.MEGABYTES): 75.39\n",
11741212
"==========================================================================================\n",
11751213
"Input size (MB): 0.20\n",
1176-
"Forward/backward pass size (MB): 12.82\n",
1177-
"Params size (MB): 0.44\n",
1178-
"Estimated Total Size (MB): 13.46\n",
1214+
"Forward/backward pass size (MB): 12.89\n",
1215+
"Params size (MB): 0.71\n",
1216+
"Estimated Total Size (MB): 13.80\n",
11791217
"==========================================================================================\n",
11801218
"{'_target_': 'nimrod.models.conv.ConvNet', 'n_features': [1, 8, 16, 32, 64, 128], 'num_classes': 10, 'kernel_size': 3, 'bias': False, 'normalization': {'_target_': 'hydra.utils.get_class', 'path': 'torch.nn.BatchNorm2d'}, 'activation': {'_target_': 'hydra.utils.get_class', 'path': 'torch.nn.ReLU'}}\n"
11811219
]
@@ -1189,16 +1227,16 @@
11891227
"\n",
11901228
"# model instantiation\n",
11911229
"convnet = ConvNet(\n",
1192-
" n_features=[1, 8, 16, 32, 64, 128], # channel/feature expansion\n",
1193-
" num_classes=10, # num_classes\n",
1194-
" kernel_size=3, # kernel size\n",
1195-
" bias=False, # conv2d bias\n",
1196-
" normalization=nn.BatchNorm2d, # normalization (before activation)\n",
1197-
" activation=nn.ReLU,\n",
1230+
" n_features=[1, 8, 16, 32, 64, 128, 64], # channel/feature expansion\n",
1231+
" num_classes=10, # num_classes\n",
1232+
" kernel_size=3, # kernel size\n",
1233+
" bias=False, # conv2d bias\n",
1234+
" normalization=nn.BatchNorm2d, # normalization (before activation)\n",
1235+
" activation=nn.ReLU,\n",
11981236
")\n",
11991237
"out = convnet(X)\n",
12001238
"print(out.shape)\n",
1201-
"print(summary(convnet, input_size=(X.shape), depth=5))\n",
1239+
"print(summary(convnet, input_size=(X.shape), depth=2))\n",
12021240
"# from config\n",
12031241
"cfg = OmegaConf.load('../config/model/image/convnet.yaml')\n",
12041242
"# print(cfg.defaults)\n",

‎nbs/models.core.ipynb

+1-1
Original file line numberDiff line numberDiff line change
@@ -474,7 +474,7 @@
474474
"\n",
475475
" if model_summary:\n",
476476
" xb, yb = next(iter(datamodule.train_dataloader()))\n",
477-
" print(summary(model.nnet, input_size=xb.shape, depth=-1, device='cpu'))\n",
477+
" print(summary(model.nnet, input_size=xb.shape, depth=5, device='cpu'))\n",
478478
" \n",
479479
" trainer.fit(model, datamodule.train_dataloader(), datamodule.val_dataloader())\n",
480480
" if test:\n",

0 commit comments

Comments
 (0)