Skip to content

Commit d7fe052

Browse files
slegrouxslegroux
committed
add logs for debug
1 parent e5f8638 commit d7fe052

File tree

4 files changed

+147
-22
lines changed

4 files changed

+147
-22
lines changed

‎nbs/image.datasets.ipynb

+129-12
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,19 @@
6767
"cell_type": "code",
6868
"execution_count": null,
6969
"metadata": {},
70-
"outputs": [],
70+
"outputs": [
71+
{
72+
"name": "stderr",
73+
"output_type": "stream",
74+
"text": [
75+
"Seed set to 42\n"
76+
]
77+
}
78+
],
7179
"source": [
7280
"#| export\n",
7381
"set_seed(42)\n",
82+
"logging.basicConfig(level=logging.DEBUG)\n",
7483
"logger = logging.getLogger(__name__)\n",
7584
"plt.set_loglevel('INFO')"
7685
]
@@ -144,7 +153,45 @@
144153
"cell_type": "code",
145154
"execution_count": null,
146155
"metadata": {},
147-
"outputs": [],
156+
"outputs": [
157+
{
158+
"data": {
159+
"text/markdown": [
160+
"---\n",
161+
"\n",
162+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/image/datasets.py#L47){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
163+
"\n",
164+
"### ImageDataset.show_idx\n",
165+
"\n",
166+
"> ImageDataset.show_idx (index:int)\n",
167+
"\n",
168+
"*display image from data point index of a image dataset*\n",
169+
"\n",
170+
"| | **Type** | **Details** |\n",
171+
"| -- | -------- | ----------- |\n",
172+
"| index | int | Index of the (image,label) sample to visualize |"
173+
],
174+
"text/plain": [
175+
"---\n",
176+
"\n",
177+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/image/datasets.py#L47){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
178+
"\n",
179+
"### ImageDataset.show_idx\n",
180+
"\n",
181+
"> ImageDataset.show_idx (index:int)\n",
182+
"\n",
183+
"*display image from data point index of a image dataset*\n",
184+
"\n",
185+
"| | **Type** | **Details** |\n",
186+
"| -- | -------- | ----------- |\n",
187+
"| index | int | Index of the (image,label) sample to visualize |"
188+
]
189+
},
190+
"execution_count": null,
191+
"metadata": {},
192+
"output_type": "execute_result"
193+
}
194+
],
148195
"source": [
149196
"show_doc(ImageDataset.show_idx)"
150197
]
@@ -188,16 +235,23 @@
188235
" # torchvision.transforms.Compose([torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(0.1307,), (0.3081,))])\n",
189236
"\n",
190237
" ):\n",
238+
" logger.info(\"MNISTDataset: init\")\n",
239+
" abs_data_dir = os.path.abspath(data_dir)\n",
240+
" logger.info(f\"Data directory: {abs_data_dir}\")\n",
191241
" os.makedirs(data_dir, exist_ok=True)\n",
192242
" super().__init__()\n",
193-
" logger.info(\"MNISTDataset: init\")\n",
194-
"\n",
195-
" self.ds = MNIST(\n",
196-
" data_dir,\n",
197-
" train = train,\n",
198-
" transform=transform, \n",
199-
" download=True\n",
200-
" )\n",
243+
" \n",
244+
" try:\n",
245+
" self.ds = MNIST(\n",
246+
" data_dir,\n",
247+
" train = train,\n",
248+
" transform=transform, \n",
249+
" download=True\n",
250+
" )\n",
251+
" logger.info(f\"MNIST dataset loaded with total {len(self.ds)} samples\")\n",
252+
" except Exception as e:\n",
253+
" logger.error(f\"Error loading MNIST dataset: {e}\")\n",
254+
" raise\n",
201255
"\n",
202256
" def __len__(self) -> int: # length of dataset\n",
203257
" return len(self.ds)\n",
@@ -228,7 +282,43 @@
228282
"cell_type": "code",
229283
"execution_count": null,
230284
"metadata": {},
231-
"outputs": [],
285+
"outputs": [
286+
{
287+
"data": {
288+
"text/markdown": [
289+
"---\n",
290+
"\n",
291+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/image/datasets.py#L131){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
292+
"\n",
293+
"### MNISTDataset.train_dev_split\n",
294+
"\n",
295+
"> MNISTDataset.train_dev_split (ratio:float)\n",
296+
"\n",
297+
"| | **Type** | **Details** |\n",
298+
"| -- | -------- | ----------- |\n",
299+
"| ratio | float | percentage of train/dev split, |\n",
300+
"| **Returns** | **tuple** | **train and set mnnist datasets** |"
301+
],
302+
"text/plain": [
303+
"---\n",
304+
"\n",
305+
"[source](https://github.com/slegroux/nimrod/blob/main/nimrod/image/datasets.py#L131){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
306+
"\n",
307+
"### MNISTDataset.train_dev_split\n",
308+
"\n",
309+
"> MNISTDataset.train_dev_split (ratio:float)\n",
310+
"\n",
311+
"| | **Type** | **Details** |\n",
312+
"| -- | -------- | ----------- |\n",
313+
"| ratio | float | percentage of train/dev split, |\n",
314+
"| **Returns** | **tuple** | **train and set mnnist datasets** |"
315+
]
316+
},
317+
"execution_count": null,
318+
"metadata": {},
319+
"output_type": "execute_result"
320+
}
321+
],
232322
"source": [
233323
"show_doc(MNISTDataset.train_dev_split)"
234324
]
@@ -245,9 +335,36 @@
245335
"cell_type": "code",
246336
"execution_count": null,
247337
"metadata": {},
248-
"outputs": [],
338+
"outputs": [
339+
{
340+
"data": {
341+
"text/html": [
342+
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">[11:01:29] </span><span style=\"color: #000080; text-decoration-color: #000080\">INFO </span> <span style=\"font-weight: bold\">[</span><span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">11:01:29</span><span style=\"font-weight: bold\">]</span> <a href=\"file:///var/folders/b5/v9y3kpzs29g41d99xvrdp3yr0000gn/T/ipykernel_81728/3839565024.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">3839565024.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///var/folders/b5/v9y3kpzs29g41d99xvrdp3yr0000gn/T/ipykernel_81728/3839565024.py#18\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">18</span></a>\n",
343+
"</pre>\n"
344+
],
345+
"text/plain": [
346+
"\u001b[2;36m[11:01:29]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m \u001b[1m[\u001b[0m\u001b[1;92m11:01:29\u001b[0m\u001b[1m]\u001b[0m \u001b]8;id=234053;file:///var/folders/b5/v9y3kpzs29g41d99xvrdp3yr0000gn/T/ipykernel_81728/3839565024.py\u001b\\\u001b[2m3839565024.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=146316;file:///var/folders/b5/v9y3kpzs29g41d99xvrdp3yr0000gn/T/ipykernel_81728/3839565024.py#18\u001b\\\u001b[2m18\u001b[0m\u001b]8;;\u001b\\\n"
347+
]
348+
},
349+
"metadata": {},
350+
"output_type": "display_data"
351+
},
352+
{
353+
"ename": "UnboundLocalError",
354+
"evalue": "cannot access local variable 'abs_data_dir' where it is not associated with a value",
355+
"output_type": "error",
356+
"traceback": [
357+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
358+
"\u001b[0;31mUnboundLocalError\u001b[0m Traceback (most recent call last)",
359+
"Cell \u001b[0;32mIn[10], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# define test set (train=False)\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m test \u001b[38;5;241m=\u001b[39m \u001b[43mMNISTDataset\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m../data/image\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtransform\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtransforms\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mToTensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;66;03m# output ( (C,H,W), int)\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28mprint\u001b[39m(test\u001b[38;5;241m.\u001b[39mds, test\u001b[38;5;241m.\u001b[39mds[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mdtype, \u001b[38;5;28mtype\u001b[39m(test\u001b[38;5;241m.\u001b[39mds[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;241m1\u001b[39m]))\n",
360+
"Cell \u001b[0;32mIn[8], line 19\u001b[0m, in \u001b[0;36mMNISTDataset.__init__\u001b[0;34m(self, data_dir, train, transform)\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 8\u001b[0m data_dir:\u001b[38;5;28mstr\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../data/image\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;66;03m# path where data is saved\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 16\u001b[0m \n\u001b[1;32m 17\u001b[0m ):\n\u001b[1;32m 18\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMNISTDataset: init\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 19\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mData directory: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mabs_data_dir\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 20\u001b[0m os\u001b[38;5;241m.\u001b[39mmakedirs(data_dir, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 21\u001b[0m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m()\n",
361+
"\u001b[0;31mUnboundLocalError\u001b[0m: cannot access local variable 'abs_data_dir' where it is not associated with a value"
362+
]
363+
}
364+
],
249365
"source": [
250366
"# define test set (train=False)\n",
367+
"\n",
251368
"test = MNISTDataset('../data/image', train=False, transform=transforms.ToTensor())\n",
252369
"\n",
253370
"# output ( (C,H,W), int)\n",

‎nbs/utils.ipynb

+1-1
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@
150150
"\n",
151151
"FORMAT = \"%(asctime)s\"\n",
152152
"logging.basicConfig(\n",
153-
" level=logging.INFO, format=FORMAT, datefmt=\"[%X]\",\n",
153+
" level=logging.DEBUG, format=FORMAT, datefmt=\"[%X]\",\n",
154154
" handlers=[RichHandler(rich_tracebacks=True, tracebacks_suppress=[matplotlib, L])]\n",
155155
")\n",
156156
"# Create a logger\n",

‎nimrod/image/datasets.py

+16-8
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
# %% ../../nbs/image.datasets.ipynb 4
3434
set_seed(42)
35+
logging.basicConfig(level=logging.DEBUG)
3536
logger = logging.getLogger(__name__)
3637
plt.set_loglevel('INFO')
3738

@@ -101,16 +102,23 @@ def __init__(
101102
# torchvision.transforms.Compose([torchvision.transforms.ToTensor(), torchvision.transforms.Normalize(0.1307,), (0.3081,))])
102103

103104
):
105+
logger.info("MNISTDataset: init")
106+
abs_data_dir = os.path.abspath(data_dir)
107+
logger.info(f"Data directory: {abs_data_dir}")
104108
os.makedirs(data_dir, exist_ok=True)
105109
super().__init__()
106-
logger.info("MNISTDataset: init")
107-
108-
self.ds = MNIST(
109-
data_dir,
110-
train = train,
111-
transform=transform,
112-
download=True
113-
)
110+
111+
try:
112+
self.ds = MNIST(
113+
data_dir,
114+
train = train,
115+
transform=transform,
116+
download=True
117+
)
118+
logger.info(f"MNIST dataset loaded with total {len(self.ds)} samples")
119+
except Exception as e:
120+
logger.error(f"Error loading MNIST dataset: {e}")
121+
raise
114122

115123
def __len__(self) -> int: # length of dataset
116124
return len(self.ds)

‎nimrod/utils.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ def set_seed(seed: int = 42) -> None:
3838

3939
FORMAT = "%(asctime)s"
4040
logging.basicConfig(
41-
level=logging.INFO, format=FORMAT, datefmt="[%X]",
41+
level=logging.DEBUG, format=FORMAT, datefmt="[%X]",
4242
handlers=[RichHandler(rich_tracebacks=True, tracebacks_suppress=[matplotlib, L])]
4343
)
4444
# Create a logger

0 commit comments

Comments
 (0)