Merge pull request #36 from caracal-pipeline/ddf-kms

Experimental ddf kms support

Author: o-smirnov

README.md

@@ -49,3 +49,14 @@ This makes the ``build-cargo.py`` script available. The script is preconfigured
 The ``cultcargo`` folder contains YaML files with cab definitions.
 
 If you would like to maintain your own image collection, write your own manifest and Dockerfiles following the cult-cargo example, and use the ``build-cargo.py`` script to build your images.
+
+## Using cult-cargo as a standalone image repository
+
+You don't even need to run stimela (or indeed install anything) to take advantage of the images packaged with cult-cargo. Take a look at the image repository on https://quay.io/organization/stimela2 to see what's available.
+
+For example, if you want to run a wsclean image, just do:
+
+```
+$ singularity build wsclean-3.3.sif docker:quay.io/stimela2/wsclean:3.3-cc0.1.2
+$ singularity exec wsclean-3.3.sif wsclean 
+```

cultcargo/builder/build_cargo.py

@@ -69,10 +69,12 @@ def run(command, cwd=None, input=None):
 @click.option('-p', '--push', is_flag=True, help='Push only, do not build.')
 @click.option('-r', '--rebuild', is_flag=True, help='Ignore docker image caches (i.e. rebuild).')
 @click.option('-a', '--all', is_flag=True, help='Build and/or push all images in manifest.')
+@click.option('-E', '--experimental', is_flag=True, help='Enable experimental versions.')
 @click.option('-v', '--verbose', is_flag=True, help='Be verbose.')
 @click.option('--boring', is_flag=True, help='Be boring -- no progress bar.')
 @click.argument('imagenames', type=str, nargs=-1)
-def build_cargo(manifest: str, do_list=False, build=False, push=False, all=False, rebuild=False, boring=False, verbose=False, imagenames: List[str] = []):
+def build_cargo(manifest: str, do_list=False, build=False, push=False, all=False, rebuild=False, boring=False,
+                experimental=False, verbose=False, imagenames: List[str] = []):
     if not (build or push or do_list):
         build = push = True
 
@@ -252,6 +254,20 @@ def resolve_config_reference(value):
                 version_vars["VERSION"] = version
                 version_vars["IMAGE_VERSION"] = image_version
 
+                is_exp = version_info.get('experimental')
+                exp_deps = version_info.get('experimental_dependencies', [])
+
+                if is_exp or exp_deps:
+                    if not experimental:
+                        print(f"[bold]{image}:{image_version}[/bold] is experimental and -E switch not given, skipping")
+                        continue
+                    # check dependencies
+                    print(f"[bold]{image}:{image_version}[/bold] is experimental")
+                    for dep in exp_deps:
+                        if not os.path.exists(dep):
+                            print(f"  [red]ERROR: dependency {dep} doesn't exist[/red]")
+                            return 1
+
                 dockerfile = version_info.get('dockerfile') or image_info.dockerfile or 'Dockerfile'
                 dockerfile = dockerfile.format(**version_vars)
                 full_image = f"{registry}/{image}:{image_version}"

cultcargo/builder/cargo-manifest.yml

@@ -54,6 +54,7 @@ images:
 
   python-astro:
     versions:
+      '3.10': {}
       '3.9': {}
       '3.8': {}
 
@@ -157,6 +158,25 @@ images:
       latest:
         package: git+https://github.com/ratt-ru/pfb-clean@awskube
 
+  ddfacet:
+    assign:
+      # base image for generic Python-based packages
+      base_python_image: python-astro:3.10
+      # corresponding python binary
+      python: python3.10
+    versions:
+      # experimental branch, Oleg only
+      useapp-meerklass:
+        experimental_dependencies:
+          - /home/oms/projects/cult-cargo/cultcargo/images/ddfacet/DDFacet-useapp-meerklass
+          - /home/oms/projects/cult-cargo/cultcargo/images/ddfacet/killMS-useapp-meerklass
+        pre_install: |
+          ADD DDFacet-useapp-meerklass DDFacet-branch
+          ADD killMS-useapp-meerklass killMS-branch
+        package: -e ./DDFacet-branch ./killMS-branch
+      '0.8':
+        package: DDFacet==0.8.0.0 killMS==3.2.0
+
   mosaic-queen:
     assign:
       pre_install: RUN apt update && apt install -y montage && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/archives

cultcargo/images/ddfacet/Dockerfile

@@ -0,0 +1,49 @@
+FROM {REGISTRY}/{base_python_image}-{BUNDLE_VERSION}
+
+RUN apt-get update && \
+    apt-get -y install \
+        dpkg-dev \
+        libc-dev \
+        cmake \
+        git \
+        wget \
+        subversion \
+        rsync \
+        python3-virtualenv \
+        python3-pip \
+        libfftw3-dev \
+        python3-numpy \
+        libfreetype6 \
+        libfreetype6-dev \
+        libpng-dev \
+        pkg-config \
+        python3-dev \
+        libboost-all-dev \
+        libcfitsio-dev \
+        libhdf5-dev \
+        wcslib-dev \
+        libatlas-base-dev \
+        liblapack-dev \
+        python3-tk \
+        libreadline6-dev \
+        subversion \
+        liblog4cplus-dev \
+        libhdf5-dev \
+        libncurses5-dev \
+        flex \
+        bison \
+        libbison-dev \
+        libqdbm-dev \
+        libgsl-dev \
+        make \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /var/cache/apt/archives 
+
+{pre_install}
+
+RUN {python} -mpip install --use-pep517 --no-cache-dir -U matplotlib dask-ms python-casacore casadata 
+RUN {python} -mpip install --use-pep517 --no-cache-dir {package} {extra_deps}
+
+{post_install}
+
+CMD ["DDF.py"]

cultcargo/images/ddfacet/apt.sources.list

@@ -0,0 +1,12 @@
+###### Ubuntu Main Repos
+deb http://us.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse
+deb-src http://us.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse
+###### Ubuntu Update Repos
+deb http://us.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse
+deb http://us.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse
+deb http://us.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse
+deb http://us.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse
+deb-src http://us.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse
+deb-src http://us.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse
+deb-src http://us.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse
+deb-src http://us.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse

cultcargo/images/killms/Dockerfile

@@ -0,0 +1,9 @@
+FROM {REGISTRY}/ddfacet:{BUNDLE_VERSION}
+
+{pre_install}
+
+RUN {python} -mpip install --use-pep517 --no-cache-dir {package} {extra_deps}
+
+{post_install}
+
+CMD ["kMS.py"]

cultcargo/images/python-astro/Dockerfile

@@ -2,9 +2,11 @@ FROM {REGISTRY}/base-cult:base-{BUNDLE_VERSION}
 
 ENV DEBIAN_FRONTEND=noninteractive
 
+RUN apt-add-repository -y ppa:deadsnakes/ppa
+
 RUN apt-get update && apt-get install apt-utils && \
-    apt-get install -y casacore-data casacore-tools python{VERSION}-dev \
-    python3-pip python3-pytest python3-matplotlib \
+    apt-get install -y casacore-data casacore-tools python{VERSION}-dev python{VERSION}-distutils \
+    python3-pip python3-pytest python3-matplotlib curl\
     && rm -rf /var/lib/apt/lists/* \
     && rm -rf /var/cache/apt/archives 
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cult-cargo"
-version = "0.1.2"
+version = "0.1.2rc8"
 description = "Curated cargo of standard radio astronomy packages for stimela 2.0"
 authors = ["Oleg Smirnov, Sphesihle Makhathini"]
 license = "MIT"