Shared tc dir

.taskcluster.yml

@@ -7,8 +7,17 @@ tasks:
     - $let:
           taskgraph:
               branch: taskgraph
-              revision: 2b32db7d303dd175ed4c4abf5fcfe00c6c593acd
+              revision: b34543ca2f42771117fa9da33c31994913587636
+          template:
+              # XXX uncomment to merge
+              # repo: https://github.com/mozilla-releng/xpi-template
+              # branch: master
+              repo: https://github.com/escapewindow/xpi-template
+              branch: shared-tc-dir
           trustDomain: xpi
+          # XXX for private repos, uncomment this line, and comment out the next
+          # githubCloneSecret: project/xpi/xpi-github-clone-ssh
+          githubCloneSecret: ""
       in:
           $if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]'
           then:
@@ -173,12 +182,18 @@ tasks:
                                     XPI_HEAD_REF: '${head_branch}'
                                     XPI_HEAD_REV: '${head_sha}'
                                     XPI_REPOSITORY_TYPE: git
+                                    TEMPLATE_BASE_REPOSITORY: '${template.repo}'
+                                    TEMPLATE_HEAD_REPOSITORY: '${template.repo}'
+                                    TEMPLATE_HEAD_REV: '${template.branch}'
+                                    TEMPLATE_HEAD_REF: '${template.branch}'
+                                    TEMPLATE_REPOSITORY_TYPE: git
                                     TASKGRAPH_BASE_REPOSITORY: https://hg.mozilla.org/ci/taskgraph
                                     TASKGRAPH_HEAD_REPOSITORY: https://hg.mozilla.org/ci/${taskgraph.branch}
                                     TASKGRAPH_HEAD_REV: '${taskgraph.revision}'
                                     TASKGRAPH_REPOSITORY_TYPE: hg
-                                    REPOSITORIES: {$json: {xpi: "XPI Manifest", taskgraph: "Taskgraph"}}
+                                    REPOSITORIES: {$json: {xpi: "XPI Manifest", taskgraph: "Taskgraph", template: "XPI Template"}}
                                     HG_STORE_PATH: /builds/worker/checkouts/hg-store
+                                    GITHUB_CLONE_SECRET: '${githubCloneSecret}'
                                   - $if: 'tasks_for in ["github-pull-request"]'
                                     then:
                                         XPI_PULL_REQUEST_NUMBER: '${event.pull_request.number}'
@@ -201,6 +216,7 @@ tasks:
                           command:
                               - /usr/local/bin/run-task
                               - '--xpi-checkout=/builds/worker/checkouts/src'
+                              - '--template-checkout=/builds/worker/checkouts/template'
                               - '--taskgraph-checkout=/builds/worker/checkouts/taskgraph'
                               - '--task-cwd=/builds/worker/checkouts/src'
                               - '--'
@@ -215,10 +231,12 @@ tasks:
                                         cd /builds/worker/checkouts/src &&
                                         ln -s /builds/worker/artifacts artifacts &&
                                         ~/.local/bin/taskgraph action-callback
+                                        --root=/builds/worker/checkouts/taskgraph/taskcluster/ci
                                     else: >
                                         PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
                                         ln -s /builds/worker/artifacts artifacts &&
                                         ~/.local/bin/taskgraph decision
+                                        --root=/builds/worker/checkouts/taskgraph/taskcluster/ci
                                         --pushlog-id='0'
                                         --pushdate='0'
                                         --project='${project}'

taskcluster/ci/build/kind.yml

@@ -19,6 +19,8 @@ job-defaults:
         docker-image:
             indexed: xpi.cache.level-3.docker-images.v2.node-latest.latest
     run:
+        checkout:
+            xpi: {}
         use-caches: false
 
 jobs:

taskcluster/ci/config.yml

@@ -8,6 +8,16 @@ taskgraph:
     repositories:
         xpi:
             name: "XPI source"
+            project-regex: '(?!xpi-template)'
+        template:
+            name: "XPI template"
+            project-regex: 'xpi-template'
+            # XXX
+            #default-repository: https://github.com/mozilla-extensions/xpi-template
+            #default-ref: master
+            default-repository: https://github.com/escapewindow/xpi-template
+            default-ref: shared-tc-dir
+            type: git
 
 workers:
     aliases:
@@ -16,6 +26,3 @@ workers:
             implementation: docker-worker
             os: linux
             worker-type: 'b-linux'
-
-# Uncomment if this is a private repo
-# github_clone_secret: project/xpi/xpi-github-clone-ssh

taskcluster/ci/docker-image/kind.yml

@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.docker_image:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+jobs:
+    node-latest:
+        definition: node
+        args:
+            NODE_VERSION: "latest"

taskcluster/ci/test/kind.yml

@@ -19,6 +19,8 @@ job-defaults:
         docker-image:
             indexed: xpi.cache.level-3.docker-images.v2.node-latest.latest
     run:
+        checkout:
+            xpi: {}
         use-caches: false
 
 jobs:

taskcluster/docker/REGISTRY

@@ -0,0 +1 @@
+mozilla-mobile

taskcluster/docker/base/Dockerfile

@@ -0,0 +1,64 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FROM ubuntu:18.04
+
+MAINTAINER Tom Prince "mozilla@hocat.ca"
+
+# Add worker user
+RUN mkdir /builds && \
+    useradd -d /builds/worker -s /bin/bash -m worker && \
+    chown worker:worker /builds/worker && \
+    mkdir /builds/worker/artifacts && \
+    chown worker:worker /builds/worker/artifacts
+
+WORKDIR /builds/worker/
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- Configuration -----------------------------------------------------------------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+ENV LANG en_US.UTF-8
+
+# Do not use fancy output on taskcluster
+ENV TERM dumb
+
+#----------------------------------------------------------------------------------------------------------------------
+#-- System ------------------------------------------------------------------------------------------------------------
+#----------------------------------------------------------------------------------------------------------------------
+
+RUN apt-get update -qq \
+    # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog that
+    # we cannot navigate while building the Docker image.
+    && apt-get install -y tzdata \
+    && apt-get install -y openjdk-8-jdk \
+                          wget \
+                          expect \
+                          git \
+                          curl \
+                          python \
+                          python-pip \
+                          python3 \
+                          locales \
+                          unzip \
+			  mercurial \
+    && apt-get clean
+
+RUN pip install --upgrade pip
+RUN pip install 'taskcluster>=4,<5'
+
+RUN locale-gen en_US.UTF-8
+
+# %include-run-task
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+# run-task expects to run as root
+USER root

taskcluster/docker/node/Dockerfile

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# %ARG           NODE_VERSION
+FROM          node:$NODE_VERSION
+MAINTAINER    Aki Sasaki <aki@mozilla.com>
+
+RUN echo "deb http://deb.debian.org/debian testing main" >> /etc/apt/sources.list
+RUN apt-get update -qq \
+    && apt-get remove -y python3 \
+    && apt-get install -y zip \
+    && apt-get install -y python3.7 \
+    && apt-get clean
+
+RUN rm /usr/bin/python3 && ln -s /usr/bin/python3.7 /usr/bin/python3
+
+# Add worker user
+RUN mkdir /builds && \
+    useradd -d /builds/worker -s /bin/bash -m worker -g 1000 -o -u 1000 && \
+    groupadd -g 1000 -o worker && \
+    mkdir /builds/worker/artifacts && \
+    chown worker:worker /builds/worker/artifacts
+
+# %include-run-task
+
+COPY build.py /usr/local/bin/build.py
+COPY test.py /usr/local/bin/test.py
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]

taskcluster/docker/node/build.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import functools
+import glob
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+
+def test_is_subdir(parent_dir, target_dir):
+    p1 = Path(os.path.realpath(parent_dir))
+    p2 = Path(os.path.realpath(target_dir))
+    if p1 not in p2.parents:
+        raise Exception("{} is not under {}!".format(target_dir, parent_dir))
+
+
+def test_var_set(varnames):
+    """Test for `varnames` in `os.environ`"""
+    errors = []
+    for varname in varnames:
+        if varname not in os.environ:
+            errors.append(("error: {} is not set".format(varname)))
+    if errors:
+        print("\n".join(errors))
+        sys.exit(1)
+
+
+def run_command(command, **kwargs):
+    print("Running {} ...".format(command))
+    subprocess.check_call(command, **kwargs)
+
+
+def get_output(command, **kwargs):
+    print("Getting output from {} ...".format(command))
+    return subprocess.check_output(command, **kwargs)
+
+
+def get_package_info():
+    if not os.path.exists("package.json"):
+        raise Exception("Can't find package.json in {}!".format(os.getcwd()))
+    with open("package.json") as fh:
+        contents = json.load(fh)
+    return contents
+
+
+def cd(path):
+    print("Changing directory to {} ...".format(path))
+    os.chdir(path)
+
+
+def mkdir(path):
+    print("mkdir {}".format(path))
+    os.makedirs(path, exist_ok=True)
+
+
+def get_hash(path, hash_alg="sha256"):
+    h = hashlib.new(hash_alg)
+    with open(path, "rb") as fh:
+        for chunk in iter(functools.partial(fh.read, 4096), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def main():
+    test_var_set([
+        "ARTIFACT_PREFIX",
+        "XPI_NAME",
+    ])
+
+    artifact_prefix = os.environ["ARTIFACT_PREFIX"]
+    xpi_name = os.environ["XPI_NAME"]
+    xpi_type = os.environ.get("XPI_TYPE")
+    repo_prefix = os.environ.get("REPO_PREFIX", "xpi")
+    head_repo_env_var = "{}_HEAD_REPOSITORY".format(repo_prefix.upper())
+    test_var_set([head_repo_env_var])
+
+    artifact_dir = "/builds/worker/artifacts"
+    base_src_dir = "/builds/worker/checkouts/src"
+
+    package_info = get_package_info()
+
+    revision = get_output(["git", "rev-parse", "HEAD"])
+
+    build_manifest = {
+        "name": xpi_name,
+        "addon-type": xpi_type,
+        "repo": os.environ[head_repo_env_var],
+        "revision": revision.decode("utf-8").rstrip(),
+        "directory": os.path.relpath(base_src_dir, os.getcwd()),
+        "version": package_info["version"],
+        "artifacts": [],
+    }
+
+    if os.environ.get("XPI_INSTALL_TYPE", "yarn") == "yarn":
+        run_command(["yarn", "install", "--frozen-lockfile"])
+    else:
+        run_command(["npm", "install"])
+
+    run_command(["yarn", "build"])
+
+    if 'XPI_ARTIFACTS' in os.environ:
+        xpi_artifacts = os.environ["XPI_ARTIFACTS"].split(";")
+    else:
+        xpi_artifacts = glob.glob('*.xpi')
+
+    for artifact in xpi_artifacts:
+        target_path = os.path.join(artifact_dir, os.path.basename(artifact))
+        if not os.path.exists(artifact):
+            raise Exception("Missing artifact {}".format(artifact))
+        test_is_subdir(os.getcwd(), artifact)
+        print("Copying {} to {}".format(artifact, target_path))
+        path = os.path.join(artifact_prefix, os.path.basename(target_path))
+        artifact_info = {
+            "path": os.path.join(artifact_prefix, os.path.basename(artifact)),
+            "filesize_bytes": int(os.path.getsize(artifact)),
+            "sha256": str(get_hash(artifact)),
+        }
+        build_manifest["artifacts"].append(artifact_info)
+        shutil.copyfile(artifact, target_path)
+
+    with open(os.path.join(artifact_dir, "manifest.json"), "w") as fh:
+        fh.write(json.dumps(build_manifest, indent=2, sort_keys=True))
+
+
+__name__ == '__main__' and main()