68 blip image captioning

.devcontainer/Dockerfile

@@ -1,41 +1,43 @@
-FROM mambaorg/micromamba:jammy-cuda-12.1.0
+# FROM mambaorg/micromamba:jammy-cuda-12.1.0
+# FROM pytorch/pytorch:2.4.0-cuda11.8-cudnn9-devel
+# torch 2.4.0 torchaudio 2.4.0 torchtext 0.15.0
+FROM bitnami/pytorch:latest 
+
 LABEL maintainer="Sylvain Le Groux <sylvain.legroux@gmail.com>"
 USER root
+
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
-     apt-get install -y --no-install-recommends \
-     build-essential \
-     libboost-all-dev \
-     libeigen3-dev \
-     cmake \
-     zlib1g-dev libbz2-dev liblzma-dev \
-     festival \
-     espeak-ng \
-     mbrola \
-     git \
-     curl \
-     ca-certificates \     
-     wget \
-     pkg-config && \
-     # Remove the effect of `apt-get update`
+    apt-get install -y --no-install-recommends \
+    build-essential \
+#     libboost-all-dev \
+#     libeigen3-dev \
+#     cmake \
+#     zlib1g-dev libbz2-dev liblzma-dev \
+#     festival \
+    espeak-ng \
+#     mbrola \
+    git \
+    git-lfs \
+    curl \
+    ca-certificates \     
+    wget \
+    pkg-config && \
+    # Remove the effect of `apt-get update`
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN git clone https://github.com/kpu/kenlm && \
-    cd kenlm; mkdir build; cd build && \
-    cmake ..; make -j 4; make install
-
-USER $MAMBA_USER
-
-WORKDIR /home/$MAMBA_USER
-COPY environment.yml .
-COPY .devcontainer/start_jupyterlab.sh .
-
-RUN micromamba install --yes --name base -f environment.yml && \
-    micromamba clean --all --yes
-ARG MAMBA_DOCKERFILE_ACTIVATE=1
-RUN python -m spacy download en_core_web_sm
-RUN nbdev_install_quarto && git clone https://github.com/slegroux/nimrod
-WORKDIR /home/$MAMBA_USER/nimrod
+# RUN git clone https://github.com/kpu/kenlm && \
+#     cd kenlm; mkdir build; cd build && \
+#     cmake ..; make -j 4; make install
+# taking care of dependencies between torch/torchtext & lightning/numpy
+RUN pip install jupyterlab nbdev torchvision torchaudio torchtext==0.15.2 torchdata numpy==1.26.4 lightning
+RUN pip install spacy && python -m spacy download en_core_web_sm && nbdev_install_quarto
+# RUN nbdev_install_quarto 
+# #&& git clone https://github.com/slegroux/nimrod
+# WORKDIR /workspaces/nimrod
+COPY . .
 RUN pip install -e .
-EXPOSE 8888
-CMD ["jupyter", "lab", "--allow-root", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--NotebookApp.token=''","--NotebookApp.password=''"]
+# # RUN git config --global --add safe.directory /workspaces/nimrod
+
+# EXPOSE 8888
+# CMD ["jupyter", "lab", "--allow-root", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--NotebookApp.token=''","--NotebookApp.password=''"]

.devcontainer/devcontainer.json

@@ -8,9 +8,12 @@
         "vscode": {
             "extensions": [
                 "ms-python.python",
+                "ms-python.vscode-pylance",
                 "ms-azuretools.vscode-docker",
                 "ms-vscode.makefile-tools",
-                "ms-toolsai.jupyter"
+                "ms-toolsai.jupyter",
+                "sourcegraph.cody-ai",
+                "quarto.quarto"
             ]
         }
     },

environment-macos.yml → .devcontainer/environment-macos.yml

@@ -44,3 +44,4 @@ dependencies:
     - torchtext
     - torchdata
     - torchaudio
+    - fairscale

.devcontainer/environment-nbdev.yml

@@ -0,0 +1,8 @@
+name: base
+channels:
+  - fastai
+  - pytorch
+  - conda-forge
+  - huggingface
+dependencies:
+  - nbdev

.devcontainer/environment.yml

@@ -3,11 +3,38 @@ channels:
   - fastai
   - pytorch
   - conda-forge
+  - huggingface
 dependencies:
+  - anyascii
+  - cupy
+  - datasets
+  - h5py
+  - hydra-core
+  - inflect
   - jupyterlab
+  - lightning 
+  - matplotlib
   - nbdev
-  - lightning
+  - omegaconf
+  - onnxruntime
+  - pandas
+  - portalocker
+  - pre-commit
+  - python-dotenv
+  - pytest
+  - spacy
+  - tqdm
+  - transformers
+  - wandb
   - pip:
+    - encodec
+    - hydra_colorlog
     - jupyterlab-quarto
     - torch
-    - torchvision
+    - torchvision
+    - torchtext
+    - torchaudio
+    - plum-dispatch
+    - https://github.com/kpu/kenlm/archive/master.zip
+    - lhotse
+    - phonemizer

.gitignore

@@ -147,6 +147,7 @@ logs/
 _proc/
 _docs/
 data/codeparrot
+data/en/
 token
 env-file
 Dockerfile.base

Makefile

@@ -31,4 +31,10 @@ train: ## Train the model
 	python src/train.py
 
 container:
-	docker build --network host -t slegroux/nimrod -f .devcontainer/Dockerfile .
+	docker build --network host -t slegroux/nimrod -f ".devcontainer/Dockerfile" .
+
+docker-test:
+	docker run -it --network host --rm --privileged slegroux/nimrod /bin/bash
+
+docker-run:
+	docker run -p 8888:8888 -v $(PWD):/data slegroux/nimrod

config/data/image/med_config.json

@@ -0,0 +1,21 @@
+{
+    "architectures": [
+      "BertModel"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "pad_token_id": 0,
+    "type_vocab_size": 2,
+    "vocab_size": 30524,
+    "encoder_width": 768,
+    "add_cross_attention": true   
+  }