update python version while keeping lean image

Author: slegroux

.devcontainer/Dockerfile

@@ -1,48 +1,17 @@
-# FROM mambaorg/micromamba:jammy-cuda-12.1.0
-# FROM pytorch/pytorch:2.4.0-cuda11.8-cudnn9-devel
-# because of deps with torchtext will downgrade to torch 2.0.1
-FROM bitnami/pytorch:2.3.1
-
+FROM docker.io/bitnami/pytorch:2.3.1
 LABEL maintainer="Sylvain Le Groux <sylvain.legroux@gmail.com>"
 USER root
-
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive && \
     apt-get install -y --no-install-recommends \
-    build-essential \
-#     libboost-all-dev \
-#     libeigen3-dev \
-#     cmake \
-#     zlib1g-dev libbz2-dev liblzma-dev \
-#     festival \
+    # build-essential \
     espeak-ng \
-#     mbrola \
-    git \
     git-lfs \
-    curl \
-    ca-certificates \     
-    wget \
-    pkg-config && \
-    # spacy takes forever to install (wheels not available)
-    pip install spacy && \
-    python -m spacy download en_core_web_sm && \
-    # Remove the effect of `apt-get update`
+    libsndfile-dev \
+    curl && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# RUN git clone https://github.com/kpu/kenlm && \
-#     cd kenlm; mkdir build; cd build && \
-#     cmake ..; make -j 4; make install
-
-# taking care of dependencies between torch/torchtext & lightning/numpy
-# RUN pip install torchtext torchvision torchaudio torchdata jupyterlab nbdev numpy lightning
-
-# RUN python -m spacy download en_core_web_sm
-# #&& git clone https://github.com/slegroux/nimrod
-# WORKDIR /workspaces/nimrod
-COPY . .
-RUN pip install -e .
-RUN nbdev_install_quarto && nbdev_install_hooks
-# # RUN git config --global --add safe.directory /workspaces/nimrod
-
-# EXPOSE 8888
-# CMD ["jupyter", "lab", "--allow-root", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--NotebookApp.token=''","--NotebookApp.password=''"]
+RUN pip install slg-nimrod==0.0.11
+RUN python -m spacy download en_core_web_sm
+RUN nbdev_install_hooks
+# RUN nbdev_install_quarto && \

.gitignore

@@ -148,6 +148,7 @@ _proc/
 _docs/
 data/codeparrot
 data/en/
+data
 token
 env-file
 Dockerfile.base

Makefile

@@ -30,12 +30,21 @@ test-full: ## Run all tests
 train: ## Train the model
 	python src/train.py
 
-container:
-	docker build --network host -t slegroux/nimrod -f ".devcontainer/Dockerfile" .
+docker-build:
+	# docker build --network host -t slegroux/nimrod -f ".devcontainer/Dockerfile" . --push
+	# docker buildx build --network host --platform linux/amd64,linux/arm64 -t slegroux/nimrod:latest -f ".devcontainer/Dockerfile.nimrod" .
+	docker buildx build --network host --platform linux/amd64 -t slegroux/nimrod:latest -f ".devcontainer/Dockerfile" .
 
-test-container:
+docker-build-mac:
+	docker buildx build --network host -t slegroux/nimrod-mac:latest -f ".devcontainer/Dockerfile" .
+
+docker-test:
+## opt: --privileged 
+	docker run -it --network host -v $(PWD):/app --rm slegroux/nimrod /bin/bash 
+
+docker-test-mac:
 ## opt: --privileged 
-	docker run -it --network host --rm slegroux/nimrod /bin/bash 
+	docker run -it --network host -v $(PWD):/app --rm slegroux/nimrod-mac /bin/bash 
 
 docker-run:
 	docker run -p 8888:8888 -v $(PWD):/app slegroux/nimrod

settings.ini

@@ -5,8 +5,8 @@
 ### Python library ###
 repo = nimrod
 lib_name = slg_%(repo)s
-version = 0.0.9
-min_python = 3.7
+version = 0.0.11
+min_python = 3.9
 license = apache2
 
 ### nbdev ###
@@ -48,11 +48,11 @@ status = 3
 ; requirements = lightning torch torchvision torchaudio torchtext torchdata \
 
 ; torch 2.0.1
-requirements = torchtext==0.15.2 torchvision==0.15.2 torchaudio==2.0.2 torchdata jupyterlab nbdev numpy lightning \
+requirements = torchtext torchvision torchaudio torchdata jupyterlab nbdev numpy lightning \
     anyascii datasets encodec fairscale hydra_colorlog hydra-core hydra-optuna-sweeper \
     inflect ipykernel jupyterlab jupyterlab-quarto librosa lhotse \
     matplotlib nvitop omegaconf onnxruntime optuna pandas phonemizer plum-dispatch portalocker \
-    pre-commit python-dotenv pytest spacy streamlit timm tensorboard tqdm transformers wandb
+    pre-commit python-dotenv pytest spacy==3.6 streamlit timm tensorboard tqdm transformers wandb
 
 # https://github.com/kpu/kenlm/archive/master.zip cupy