Merge pull request #1 from yacchin1205/feature/grdm

ikfj · web-flow · commit 02ee300f0427 · 2022-05-11T19:50:06.000+09:00
[GRDM-31008] GRDM対応
diff --git a/README.md b/README.md
@@ -17,14 +17,24 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
 sudo add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
 sudo apt update && sudo apt install -y docker-ce
 
+curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash -
+sudo apt update && sudo apt install -y nodejs libfuse-dev python3-dev libcurl4-openssl-dev libssl-dev
+sudo modprobe fuse
+
 # pull the repo2docker image
-sudo docker pull quay.io/jupyterhub/repo2docker:main
+sudo docker pull gcr.io/nii-ap-ops/repo2docker:20220330
+sudo docker pull gcr.io/nii-ap-ops/rdmfs:20211221
 
 # install TLJH
 curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \
   | sudo python3 - \
     --admin test:test \
-    --plugin git+https://github.com/plasmabio/tljh-repo2docker@master
+    --plugin git+https://github.com/RCOSDP/CS-tljh-repo2docker.git@master
+
+# fix to use the RCOSDP's one as binderhub package.
+sudo /opt/tljh/hub/bin/python -m pip install --upgrade git+https://github.com/RCOSDP/CS-binderhub.git
+# restart TLJH
+sudo systemctl restart jupyterhub
 ```
 
 Refer to [The Littlest JupyterHub documentation](http://tljh.jupyter.org/en/latest/topic/customizing-installer.html?highlight=plugins#installing-tljh-plugins)
diff --git a/setup.py b/setup.py
@@ -6,5 +6,10 @@
     entry_points={"tljh": ["tljh_repo2docker = tljh_repo2docker"]},
     packages=find_packages(),
     include_package_data=True,
-    install_requires=["dockerspawner~=12.1", "jupyter_client~=6.1", "aiodocker~=0.19"],
+    install_requires=[
+        "dockerspawner~=12.1", "jupyter_client~=6.1", "aiodocker~=0.19", "binderhub",
+    ],
+    dependency_links = [
+        "git+https://github.com/RCOSDP/CS-binderhub.git#egg=binderhub",
+    ],
 )
diff --git a/tljh_repo2docker/__init__.py b/tljh_repo2docker/__init__.py
@@ -1,7 +1,10 @@
 import os
 
 from aiodocker import Docker
+from aiodocker.exceptions import DockerError
 from dockerspawner import DockerSpawner
+from docker.errors import APIError
+from docker.types import Mount
 from jinja2 import Environment, BaseLoader
 from jupyter_client.localinterfaces import public_ips
 from jupyterhub.handlers.static import CacheControlStaticFilesHandler
@@ -10,11 +13,15 @@
 from tljh.configurer import load_config
 from traitlets import Unicode
 from traitlets.config import Configurable
+from tornado import web
 
 from .builder import BuildHandler
+from .launcher import LaunchHandler
 from .docker import list_images
 from .images import ImagesHandler
 from .logs import LogsHandler
+from .token import TokenStore
+
 
 # Default CPU period
 # See: https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory#configure-the-default-cfs-scheduler
@@ -92,6 +99,22 @@ class SpawnerMixin(Configurable):
         """,
     )
 
+    rdmfs_base_path = Unicode(
+        config=True,
+        help="""
+        A base path for RDMFS.
+        """,
+    )
+
+    token_store_path = Unicode(
+        config=True,
+        help="""
+        A dbpath of token_store.
+        """,
+    )
+
+    extra_mounts = None
+
     async def list_images(self):
         """
         Return the list of available images
@@ -155,19 +178,171 @@ async def set_limits(self):
                 }
             )
 
+    async def set_extra_mounts(self):
+        """
+        Prepare volume binds for GRDM
+        """
+        imagename = self.user_options.get("image")
+        async with Docker() as docker:
+            image = await docker.images.inspect(imagename)
+        
+        provider_prefix = image["ContainerConfig"]["Labels"].get(
+            "tljh_repo2docker.opt.provider", None
+        )
+        if provider_prefix != 'rdm':
+            return
+        await self._set_rdm_mounts(image)
+
+    async def _set_rdm_mounts(self, image):
+        repo = image["ContainerConfig"]["Labels"].get(
+            "tljh_repo2docker.opt.repo", None
+        )
+        token_store = TokenStore(dbpath=self.token_store_path)
+        repo_token = token_store.get(self.user, repo)
+        if repo_token is None:
+            raise web.HTTPError(
+                400,
+                "No repo_token for: %s" % (repo),
+            )
+        self.log.info("Preparing RDMFS... " + 'name=' + repr(self.user.name) + ', repo=' + repr(repo))
+        mount_path = os.path.join(self.rdmfs_base_path, self.container_name)
+        if not os.path.exists(mount_path):
+            os.makedirs(mount_path)
+        self.extra_mounts = [
+            dict(type='bind', source=mount_path, target='/mnt', propagation='rshared'),
+        ]
+        rdmfs_id = await self.get_rdmfs_object()
+        if rdmfs_id is not None:
+            await self.remove_object_by_id(rdmfs_id)
+        rdmfs_id = await self.create_rdmfs_object({
+            'RDM_NODE_ID': image["ContainerConfig"]["Labels"].get(
+                "tljh_repo2docker.opt.user.rdm_node_id", None
+            ),
+            'RDM_API_URL': image["ContainerConfig"]["Labels"].get(
+                "tljh_repo2docker.opt.user.rdm_api_url", None
+            ),
+            'RDM_TOKEN': repo_token,
+            'MOUNT_PATH': '/mnt/rdm',
+        })
+        await self.start_object_by_id(rdmfs_id)
+
+    async def get_rdmfs_object(self):
+        object_name = self.object_name + '_rdmfs'
+        self.log.debug("Getting %s '%s'", self.object_type, object_name)
+        try:
+            async with Docker() as docker:
+                obj = await docker.containers.get(object_name)
+            return obj.id
+        except DockerError as e:
+            if e.status == 404:
+                self.log.info(
+                    "%s '%s' is gone", self.object_type.title(), object_name
+                )
+            elif e.status == 500:
+                self.log.info(
+                    "%s '%s' is on unhealthy node",
+                    self.object_type.title(),
+                    object_name,
+                )
+            else:
+                raise
+        return None
+
+    async def create_rdmfs_object(self, env):
+        host_config = dict(
+            Mounts=[
+                {
+                    "Type": "bind",
+                    "Source": m['source'],
+                    "Target": "/mnt",
+                    "ReadOnly": False,
+                    "BindOptions": {
+                        "Propagation": "rshared",
+                    },
+                }
+                for m in (self.extra_mounts or [])
+            ],
+            Privileged=True,
+        )
+        create_kwargs = dict(
+            Image='gcr.io/nii-ap-ops/rdmfs:20211221',
+            Env=[f'{k}={v}' for k, v in env.items()],
+            AutoRemove=True,
+            HostConfig=host_config,
+        )
+        async with Docker() as docker:
+            obj = await docker.containers.create(
+                create_kwargs,
+                name=self.container_name + '_rdmfs',
+            )
+        return obj.id
+
+    async def start_object_by_id(self, object_id):
+        async with Docker() as docker:
+            obj = await docker.containers.get(object_id)
+            await obj.start()
+
+    async def remove_object_by_id(self, object_id):
+        self.log.info("Removing %s %s", self.object_type, object_id)
+        try:
+            async with Docker() as docker:
+                obj = await docker.containers.get(object_id)
+                desc = await obj.show()
+                if 'State' in desc and desc['State']['Running']:
+                    self.log.info('terminating...')
+                    exec = await obj.exec(["/bin/sh","-c","xattr -w command terminate /mnt/rdm"])
+                    result = await exec.start(detach=True)
+                    self.log.info('terminated: {}'.format(result))
+                else:
+                    self.log.info('deleting...')
+                    await obj.delete()
+        except DockerError as e:
+            if e.status == 409:
+                self.log.debug(
+                    "Already removing %s: %s", self.object_type, object_id
+                )
+            elif e.status == 404:
+                self.log.debug(
+                    "Already removed %s: %s", self.object_type, object_id
+                )
+            else:
+                raise
+
 
 class Repo2DockerSpawner(SpawnerMixin, DockerSpawner):
     """
     A custom spawner for using local Docker images built with tljh-repo2docker.
     """
 
+    @property
+    def mount_binds(self):
+        base_mount_binds = super().mount_binds.copy()
+        if self.extra_mounts is None:
+            return base_mount_binds
+        base_mount_binds += [Mount(**m) for m in self.extra_mounts]
+        return base_mount_binds
+
     async def start(self, *args, **kwargs):
         await self.set_limits()
+        await self.set_extra_mounts()
         return await super().start(*args, **kwargs)
 
+    async def stop(self, *args, **kwargs):
+        await super().stop(*args, **kwargs)
+        rdmfs_id = await self.get_rdmfs_object()
+        if rdmfs_id is None:
+            return
+        await self.remove_object_by_id(rdmfs_id)
+
 
 @hookimpl
 def tljh_custom_jupyterhub_config(c):
+    from binderhub.repoproviders import (
+        GitHubRepoProvider, GitRepoProvider, GitLabRepoProvider, GistRepoProvider,
+        ZenodoProvider, FigshareProvider, HydroshareProvider, DataverseProvider,
+        RDMProvider, WEKO3Provider,
+    )
+
     # hub
     c.JupyterHub.hub_ip = public_ips()[0]
     c.JupyterHub.cleanup_servers = False
@@ -178,10 +353,14 @@ def tljh_custom_jupyterhub_config(c):
         0, os.path.join(os.path.dirname(__file__), "templates")
     )
 
+    token_store_path = '/opt/tljh/state/repo2docker.sqlite'
+
     # spawner
     c.DockerSpawner.cmd = ["jupyterhub-singleuser"]
     c.DockerSpawner.pull_policy = "Never"
     c.DockerSpawner.remove = True
+    c.Repo2DockerSpawner.rdmfs_base_path = '/opt/tljh/repo2docker/volumes'
+    c.Repo2DockerSpawner.token_store_path = token_store_path
 
     # fetch limits from the TLJH config
     tljh_config = load_config()
@@ -193,12 +372,47 @@ def tljh_custom_jupyterhub_config(c):
         {"default_cpu_limit": cpu_limit, "default_mem_limit": mem_limit}
     )
 
+    repo_providers = {
+        'gh': GitHubRepoProvider,
+        'gist': GistRepoProvider,
+        'git': GitRepoProvider,
+        'gl': GitLabRepoProvider,
+        'zenodo': ZenodoProvider,
+        'figshare': FigshareProvider,
+        'hydroshare': HydroshareProvider,
+        'dataverse': DataverseProvider,
+        'rdm': RDMProvider,
+        'weko3': WEKO3Provider,
+    }
+    c.RDMProvider.hosts = [
+        {
+            'hostname': ["https://osf.io/"],
+            'api': "https://api.osf.io/v2/"
+        },
+        {
+            'hostname': ["https://bh.rdm.yzwlab.com/"],
+            'api': "https://api.bh.rdm.yzwlab.com/v2/",
+        },
+        {
+            'hostname': ["https://rcos.rdm.nii.ac.jp"],
+            'api': "https://api.rcos.rdm.nii.ac.jp/v2/",
+        },
+    ]
+    
     # register the handlers to manage the user images
     c.JupyterHub.extra_handlers.extend(
         [
             (r"environments", ImagesHandler),
             (r"api/environments", BuildHandler),
             (r"api/environments/([^/]+)/logs", LogsHandler),
+            (
+                r"build/([^/]+)/[^/]+/[^/]+",
+                LaunchHandler,
+                {
+                    "repo_providers": repo_providers,
+                    "token_store_path": token_store_path,
+                },
+            ),
             (
                 r"environments-static/(.*)",
                 CacheControlStaticFilesHandler,
@@ -210,4 +424,9 @@ def tljh_custom_jupyterhub_config(c):
 
 @hookimpl
 def tljh_extra_hub_pip_packages():
-    return ["dockerspawner~=0.11", "jupyter_client~=6.1", "aiodocker~=0.19"]
+    return [
+        "dockerspawner~=12.1",
+        "jupyter_client~=6.1",
+        "aiodocker~=0.19",
+        "git+https://github.com/RCOSDP/CS-binderhub.git",
+    ]
diff --git a/tljh_repo2docker/docker.py b/tljh_repo2docker/docker.py
@@ -56,7 +56,8 @@ async def list_containers():
 
 async def build_image(
     repo, ref, name="", memory=None, cpu=None, username=None, password=None,
-    extra_buildargs=None
+    extra_buildargs=None, repo2docker_image=None, optional_envs=None, default_image_name=None,
+    optional_labels=None
 ):
     """
     Build an image given a repo, ref and limits
@@ -67,9 +68,12 @@ async def build_image(
 
     # default to the repo name if no name specified
     # and sanitize the name of the docker image
-    name = name or urlparse(repo).path.strip("/")
-    name = name.lower().replace("/", "-")
-    image_name = f"{name}:{ref}"
+    if default_image_name is not None:
+        image_name = name = default_image_name
+    else:
+        name = name or urlparse(repo).path.strip("/")
+        name = name.lower().replace("/", "-")
+        image_name = f"{name}:{ref}"
 
     # memory is specified in GB
     memory = f"{memory}G" if memory else ""
@@ -82,6 +86,9 @@ async def build_image(
         f"tljh_repo2docker.mem_limit={memory}",
         f"tljh_repo2docker.cpu_limit={cpu}",
     ]
+    if optional_labels is not None:
+        labels += [f"tljh_repo2docker.opt.{k}={v}" for k, v in optional_labels.items()]
+
     cmd = [
         "jupyter-repo2docker",
         "--ref",
@@ -108,10 +115,14 @@ async def build_image(
         ]
 
     cmd.append(repo)
+    envs = []
+    if optional_envs is not None:
+        for k, v in optional_envs.items():
+            envs.append(f'{k}={v}')
 
     config = {
         "Cmd": cmd,
-        "Image": "quay.io/jupyterhub/repo2docker:main",
+        "Image": repo2docker_image or "quay.io/jupyterhub/repo2docker:main",
         "Labels": {
             "repo2docker.repo": repo,
             "repo2docker.ref": ref,
@@ -126,6 +137,7 @@ async def build_image(
                 "mode": "rw",
             }
         },
+        "Env": envs,
         "HostConfig": {
             "Binds": ["/var/run/docker.sock:/var/run/docker.sock"],
         },
diff --git a/tljh_repo2docker/launcher.py b/tljh_repo2docker/launcher.py
diff --git a/tljh_repo2docker/token.py b/tljh_repo2docker/token.py
