pw_package: Initial commit

Add pw_package module. This manages dependencies that aren't pulled in
through env setup. For now only nanopb is available through pw_package.

Change-Id: Ib8a20102baf27d5964bb275088c265f9334b6ff3
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/22020
Reviewed-by: Anthony DiGirolamo <tonymd@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>

Author: mohrr

PW_PLUGINS

@@ -14,3 +14,4 @@
 presubmit pw_presubmit.pigweed_presubmit main
 heap-viewer pw_allocator.heap_viewer main
 rpc pw_hdlc_lite.rpc_console main
+package pw_package.pigweed_packages main

docs/BUILD.gn

@@ -79,6 +79,7 @@ group("module_docs") {
     "$dir_pw_metric:docs",
     "$dir_pw_minimal_cpp_stdlib:docs",
     "$dir_pw_module:docs",
+    "$dir_pw_package:docs",
     "$dir_pw_polyfill:docs",
     "$dir_pw_preprocessor:docs",
     "$dir_pw_presubmit:docs",

modules.gni

@@ -51,6 +51,7 @@ declare_args() {
   dir_pw_minimal_cpp_stdlib = get_path_info("pw_minimal_cpp_stdlib", "abspath")
   dir_pw_module = get_path_info("pw_module", "abspath")
   dir_pw_fuzzer = get_path_info("pw_fuzzer", "abspath")
+  dir_pw_package = get_path_info("pw_package", "abspath")
   dir_pw_polyfill = get_path_info("pw_polyfill", "abspath")
   dir_pw_preprocessor = get_path_info("pw_preprocessor", "abspath")
   dir_pw_presubmit = get_path_info("pw_presubmit", "abspath")

pw_package/BUILD.gn

@@ -0,0 +1,21 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("//build_overrides/pigweed.gni")
+
+import("$dir_pw_docgen/docs.gni")
+
+pw_doc_group("docs") {
+  sources = [ "docs.rst" ]
+}

pw_package/docs.rst

@@ -0,0 +1,110 @@
+.. _module-pw_package:
+
+==========
+pw_package
+==========
+The package module provides a mechanism to install additional tools used by
+Pigweed. Most Pigweed dependencies should be installed using
+:ref:`module-pw_env_setup`. Examples of reasons packages should be managed using
+this module instead are listed below.
+
+* The dependency is extremely large and not commonly used.
+* The dependency has a number of compatible versions and we want to allow
+  downstream projects to pick a version rather than being forced to use ours.
+* The dependency has license issues that make it complicated for Google to
+  include it directly as a submodule or distribute it as a CIPD package.
+* The dependency needs to be "installed" into the system in some manner beyond
+  just extraction and thus isn't a good match for distribution with CIPD.
+
+-----
+Usage
+-----
+The package module can be accessed through the ``pw package`` command. This
+has several subcommands.
+
+``pw package list``
+  Lists all the packages installed followed by all the packages available.
+
+``pw package install <package-name>``
+  Installs ``<package-name>``. Exactly how this works is package-dependent,
+  and packages can decide to do nothing because the package is current, do an
+  incremental update, or delete the current version and install anew. Use
+  ``--force`` to remove the package before installing.
+
+``pw package status <package-name>``
+  Indicates whether ``<packagxe-name>`` is installed.
+
+``pw package remove <package-name>``
+  Removes ``<package-name>``.
+
+-----------
+Configuring
+-----------
+
+Compatibility
+~~~~~~~~~~~~~
+Python 3
+
+Adding a New Package
+~~~~~~~~~~~~~~~~~~~~
+To add a new package create a class that subclasses ``Package`` from
+``pw_package/package_manager.py``.
+
+.. code-block:: python
+
+  class Package:
+      """Package to be installed.
+
+      Subclass this to implement installation of a specific package.
+      """
+      def __init__(self, name):
+          self._name = name
+
+      @property
+      def name(self):
+          return self._name
+
+      def install(self, path: pathlib.Path) -> None:
+          """Install the package at path.
+
+          Install the package in path. Cannot assume this directory is empty—it
+          may need to be deleted or updated.
+          """
+
+      def remove(self, path: pathlib.Path) -> None:
+          """Remove the package from path.
+
+          Removes the directory containing the package. For most packages this
+          should be sufficient to remove the package, and subclasses should not
+          need to override this package.
+          """
+          if os.path.exists(path):
+              shutil.rmtree(path)
+
+      def status(self, path: pathlib.Path) -> bool:
+          """Returns if package is installed at path and current.
+
+          This method will be skipped if the directory does not exist.
+          """
+
+There's also a helper class for retrieving specific revisions of Git
+repositories in ``pw_package/git_repo.py``.
+
+Then call ``pw_package.package_manager.register(PackageClass)`` to register
+the class with the package manager.
+
+Setting up a Project
+~~~~~~~~~~~~~~~~~~~~
+To set up the package manager for a new project create a file like below and
+add it to the ``PW_PLUGINS`` file (see :ref:`module-pw_cli` for details). This
+file is based off of ``pw_package/pigweed_packages.py``.
+
+.. code-block:: python
+
+  from pw_package import package_manager
+  # These modules register themselves so must be imported despite appearing
+  # unused.
+  from pw_package.packages import nanopb
+
+  def main(argv=None) -> int:
+      return package_manager.run(**vars(package_manager.parse_args(argv)))

pw_package/py/pw_package/git_repo.py

@@ -0,0 +1,71 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Install and check status of Git repository-based packages."""
+
+import os
+import pathlib
+import shutil
+import subprocess
+from typing import Union
+
+import pw_package.package_manager
+
+PathOrStr = Union[pathlib.Path, str]
+
+
+def git_stdout(*args: PathOrStr,
+               show_stderr=False,
+               repo: PathOrStr = '.') -> str:
+    return subprocess.run(['git', '-C', repo, *args],
+                          stdout=subprocess.PIPE,
+                          stderr=None if show_stderr else subprocess.DEVNULL,
+                          check=True).stdout.decode().strip()
+
+
+def git(*args: PathOrStr,
+        repo: PathOrStr = '.') -> subprocess.CompletedProcess:
+    return subprocess.run(['git', '-C', repo, *args], check=True)
+
+
+class GitRepo(pw_package.package_manager.Package):
+    """Install and check status of Git repository-based packages."""
+    def __init__(self, url, commit, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._url = url
+        self._commit = commit
+
+    def status(self, path: pathlib.Path) -> bool:
+        if not os.path.isdir(path / '.git'):
+            return False
+
+        remote = git_stdout('remote', 'get-url', 'origin', repo=path)
+        commit = git_stdout('rev-parse', 'HEAD', repo=path)
+        status = git_stdout('status', '--porcelain=v1', repo=path)
+        return remote == self._url and commit == self._commit and not status
+
+    def install(self, path: pathlib.Path) -> None:
+        # If already installed and at correct version exit now.
+        if self.status(path):
+            return
+
+        # Otherwise delete current version and clone again.
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+
+        # --filter=blob:none means we don't get history, just the current
+        # revision. If we later run commands that need history it will be
+        # retrieved on-demand. For small repositories the effect is negligible
+        # but for large repositories this should be a significant improvement.
+        git('clone', '--filter=blob:none', self._url, path)
+        git('reset', '--hard', self._commit, repo=path)

pw_package/py/pw_package/package_manager.py

@@ -0,0 +1,147 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Install and remove optional packages."""
+
+import argparse
+import logging
+import os
+import pathlib
+import shutil
+from typing import List
+
+_LOG: logging.Logger = logging.getLogger(__name__)
+
+
+class Package:
+    """Package to be installed.
+
+    Subclass this to implement installation of a specific package.
+    """
+    def __init__(self, name):
+        self._name = name
+
+    @property
+    def name(self):
+        return self._name
+
+    def install(self, path: pathlib.Path) -> None:  # pylint: disable=no-self-use
+        """Install the package at path.
+
+        Install the package in path. Cannot assume this directory is empty—it
+        may need to be deleted or updated.
+        """
+
+    def remove(self, path: pathlib.Path) -> None:  # pylint: disable=no-self-use
+        """Remove the package from path.
+
+        Removes the directory containing the package. For most packages this
+        should be sufficient to remove the package, and subclasses should not
+        need to override this package.
+        """
+        if os.path.exists(path):
+            shutil.rmtree(path)
+
+    def status(self, path: pathlib.Path) -> bool:  # pylint: disable=no-self-use
+        """Returns if package is installed at path and current.
+
+        This method will be skipped if the directory does not exist.
+        """
+
+
+_PACKAGES = {}
+
+
+def register(package_class: type) -> None:
+    obj = package_class()
+    _PACKAGES[obj.name] = obj
+
+
+class PackageManager:
+    """Install and remove optional packages."""
+    def __init__(self):
+        self._pkg_root: pathlib.Path = None
+
+    def install(self, package: str, force=False):
+        pkg = _PACKAGES[package]
+        if force:
+            self.remove(package)
+        _LOG.info('Installing %s...', pkg.name)
+        pkg.install(self._pkg_root / pkg.name)
+        _LOG.info('Installing %s...done.', pkg.name)
+        return 0
+
+    def remove(self, package: str):  # pylint: disable=no-self-use
+        pkg = _PACKAGES[package]
+        _LOG.info('Removing %s...', pkg.name)
+        pkg.remove(self._pkg_root / pkg.name)
+        _LOG.info('Removing %s...done.', pkg.name)
+        return 0
+
+    def status(self, package: str):  # pylint: disable=no-self-use
+        pkg = _PACKAGES[package]
+        path = self._pkg_root / pkg.name
+        if os.path.isdir(path) and pkg.status(path):
+            _LOG.info('%s is installed.', pkg.name)
+            return 0
+
+        _LOG.info('%s is not installed.', pkg.name)
+        return -1
+
+    def list(self):  # pylint: disable=no-self-use
+        _LOG.info('Installed packages:')
+        available = []
+        for package in sorted(_PACKAGES.keys()):
+            pkg = _PACKAGES[package]
+            if pkg.status(self._pkg_root / pkg.name):
+                _LOG.info('  %s', pkg.name)
+            else:
+                available.append(pkg.name)
+        _LOG.info('')
+
+        _LOG.info('Available packages:')
+        for pkg_name in available:
+            _LOG.info('  %s', pkg_name)
+        _LOG.info('')
+
+        return 0
+
+    def run(self, command: str, pkg_root: pathlib.Path, **kwargs):
+        os.makedirs(pkg_root, exist_ok=True)
+        self._pkg_root = pkg_root
+        return getattr(self, command)(**kwargs)
+
+
+def parse_args(argv: List[str] = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser("Manage packages.")
+    parser.add_argument(
+        '--package-root',
+        '-e',
+        dest='pkg_root',
+        type=pathlib.Path,
+        default=(pathlib.Path(os.environ['_PW_ACTUAL_ENVIRONMENT_ROOT']) /
+                 'packages'),
+    )
+    subparsers = parser.add_subparsers(dest='command', required=True)
+    install = subparsers.add_parser('install')
+    install.add_argument('--force', '-f', action='store_true')
+    remove = subparsers.add_parser('remove')
+    status = subparsers.add_parser('status')
+    for cmd in (install, remove, status):
+        cmd.add_argument('package', choices=_PACKAGES.keys())
+    _ = subparsers.add_parser('list')
+    return parser.parse_args(argv)
+
+
+def run(**kwargs):
+    return PackageManager().run(**kwargs)

pw_package/py/pw_package/packages/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.