Fix CLI and install instructions in case optional checklists is not p…

…resent (#5589)

* add conda instructions for optional checklist install

* clean up python 3.6 left-overs

* create dummy command if checklist not available

* do not import private submodules in util.import_module_and_submodules

* work-around for mypy issue

* also exclude test modules in import_module_and_submodules

* exclude some more modules

* shift type ignore

* run CI against base package

* add changelog

* Revert "run CI against base package"

This reverts commit 8713936.

* test CLI against package flavors

* fix typo in changelog

Co-authored-by: Akshita Bhagia <akshita23bhagia@gmail.com>

.github/workflows/ci.yml

@@ -348,6 +348,9 @@ jobs:
     strategy:
       matrix:
         python: ['3.7', '3.8', '3.9']
+        # check that CLI remains working for all package flavors;
+        # currently allennlp[checklist]==allennlp[all], so avoiding duplication on this
+        flavor: ['', '[all]']
 
     steps:
     - uses: actions/checkout@v2
@@ -369,7 +372,7 @@ jobs:
 
     - name: Install core package
       run: |
-        pip install $(ls dist/*.whl)[all]
+        pip install $(ls dist/*.whl)${{ matrix.flavor }}
 
     - name: Download NLTK prerequisites
       run: |

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Removed unnecessary dependencies
+- Restore functionality of CLI in absence of now-optional checklist-package
 
 
 ## [v2.9.1](https://github.com/allenai/allennlp/releases/tag/v2.9.1) - 2022-03-09

README.md

@@ -179,13 +179,19 @@ We support AllenNLP on Mac and Linux environments. We presently do not support W
 
 ### Installing via conda-forge
 
-The simplest way to install AllenNLP is using conda:
+The simplest way to install AllenNLP is using conda (you can choose a different python version):
 
 ```
 conda install -c conda-forge python=3.8 allennlp
 ```
 
-All plugins mentioned above are similarly installable, e.g.
+To install optional packages, such as `checklist`, use
+
+```
+conda install -c conda-forge allennlp-checklist
+```
+
+or simply install `allennlp-all` directly. The plugins mentioned above are similarly installable, e.g.
 
 ```
 conda install -c conda-forge allennlp-models allennlp-semparse allennlp-server allennlp-optuna
@@ -211,10 +217,10 @@ environment you want to use, you can skip to the 'installing via pip' section.
 
 1.  [Download and install Conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html).
 
-2.  Create a Conda environment with Python 3.7 (3.6 or 3.8 would work as well):
+2.  Create a Conda environment with Python 3.8 (3.7 or 3.9 would work as well):
 
     ```
-    conda create -n allennlp_env python=3.7
+    conda create -n allennlp_env python=3.8
     ```
 
 3.  Activate the Conda environment. You will need to activate the Conda environment in each terminal in which you want to use AllenNLP:

allennlp/__init__.py

@@ -1,10 +1,3 @@
-# Make sure that allennlp is running on Python 3.6.1 or later
-# (to avoid running into this bug: https://bugs.python.org/issue29246)
-import sys
-
-if sys.version_info < (3, 6, 1):
-    raise RuntimeError("AllenNLP requires Python 3.6.1 or later")
-
 # We get a lot of these spurious warnings,
 # see https://github.com/ContinuumIO/anaconda-issues/issues/6678
 import warnings  # noqa

allennlp/commands/__init__.py

@@ -7,6 +7,7 @@
 from allennlp import __version__
 from allennlp.commands.build_vocab import BuildVocab
 from allennlp.commands.cached_path import CachedPath
+from allennlp.commands.checklist import CheckList
 from allennlp.commands.diff import Diff
 from allennlp.commands.evaluate import Evaluate
 from allennlp.commands.find_learning_rate import FindLearningRate
@@ -22,17 +23,6 @@
 
 logger = logging.getLogger(__name__)
 
-try:
-    """
-    The `allennlp checklist` command requires installation of the optional dependency `checklist`.
-    It can be installed with `pip install allennlp[checklist]`.
-    """
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        from allennlp.commands.checklist import CheckList
-except ImportError:
-    pass
-
 
 class ArgumentParserWithDefaults(argparse.ArgumentParser):
     """

allennlp/commands/_checklist_internal.py

@@ -0,0 +1,204 @@
+"""
+The `checklist` subcommand allows you to conduct behavioural
+testing for your model's predictions using a trained model and its
+[`Predictor`](../predictors/predictor.md#predictor) wrapper.
+"""
+
+from typing import Optional, Dict, Any, List
+import argparse
+import sys
+import json
+import logging
+
+
+from allennlp.commands.subcommand import Subcommand
+from allennlp.common.checks import check_for_gpu, ConfigurationError
+from allennlp.models.archival import load_archive
+from allennlp.predictors.predictor import Predictor
+
+logger = logging.getLogger(__name__)
+
+try:
+    from allennlp.confidence_checks.task_checklists.task_suite import TaskSuite
+except ImportError:
+    raise
+
+
+@Subcommand.register("checklist")
+class CheckList(Subcommand):
+    def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
+
+        description = """Run the specified model through a checklist suite."""
+        subparser = parser.add_parser(
+            self.name,
+            description=description,
+            help="Run a trained model through a checklist suite.",
+        )
+
+        subparser.add_argument(
+            "archive_file", type=str, help="The archived model to make predictions with"
+        )
+
+        subparser.add_argument("task", type=str, help="The name of the task suite")
+
+        subparser.add_argument("--checklist-suite", type=str, help="The checklist suite path")
+
+        subparser.add_argument(
+            "--capabilities",
+            nargs="+",
+            default=[],
+            help=('An optional list of strings of capabilities. Eg. "[Vocabulary, Robustness]"'),
+        )
+
+        subparser.add_argument(
+            "--max-examples",
+            type=int,
+            default=None,
+            help="Maximum number of examples to check per test.",
+        )
+
+        subparser.add_argument(
+            "--task-suite-args",
+            type=str,
+            default="",
+            help=(
+                "An optional JSON structure used to provide additional parameters to the task suite"
+            ),
+        )
+
+        subparser.add_argument(
+            "--print-summary-args",
+            type=str,
+            default="",
+            help=(
+                "An optional JSON structure used to provide additional "
+                "parameters for printing test summary"
+            ),
+        )
+
+        subparser.add_argument("--output-file", type=str, help="Path to output file")
+
+        subparser.add_argument(
+            "--cuda-device", type=int, default=-1, help="ID of GPU to use (if any)"
+        )
+
+        subparser.add_argument(
+            "--predictor", type=str, help="Optionally specify a specific predictor to use"
+        )
+
+        subparser.add_argument(
+            "--predictor-args",
+            type=str,
+            default="",
+            help=(
+                "An optional JSON structure used to provide additional parameters to the predictor"
+            ),
+        )
+
+        subparser.set_defaults(func=_run_suite)
+
+        return subparser
+
+
+def _get_predictor(args: argparse.Namespace) -> Predictor:
+    check_for_gpu(args.cuda_device)
+    archive = load_archive(
+        args.archive_file,
+        cuda_device=args.cuda_device,
+    )
+
+    predictor_args = args.predictor_args.strip()
+    if len(predictor_args) <= 0:
+        predictor_args = {}
+    else:
+        predictor_args = json.loads(predictor_args)
+
+    return Predictor.from_archive(
+        archive,
+        args.predictor,
+        extra_args=predictor_args,
+    )
+
+
+def _get_task_suite(args: argparse.Namespace) -> TaskSuite:
+    available_tasks = TaskSuite.list_available()
+    if args.task in available_tasks:
+        suite_name = args.task
+    else:
+        raise ConfigurationError(
+            f"'{args.task}' is not a recognized task suite. "
+            f"Available tasks are: {available_tasks}."
+        )
+
+    file_path = args.checklist_suite
+
+    task_suite_args = args.task_suite_args.strip()
+    if len(task_suite_args) <= 0:
+        task_suite_args = {}
+    else:
+        task_suite_args = json.loads(task_suite_args)
+
+    return TaskSuite.constructor(
+        name=suite_name,
+        suite_file=file_path,
+        extra_args=task_suite_args,
+    )
+
+
+class _CheckListManager:
+    def __init__(
+        self,
+        task_suite: TaskSuite,
+        predictor: Predictor,
+        capabilities: Optional[List[str]] = None,
+        max_examples: Optional[int] = None,
+        output_file: Optional[str] = None,
+        print_summary_args: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        self._task_suite = task_suite
+        self._predictor = predictor
+        self._capabilities = capabilities
+        self._max_examples = max_examples
+        self._output_file = None if output_file is None else open(output_file, "w")
+        self._print_summary_args = print_summary_args or {}
+
+        if capabilities:
+            self._print_summary_args["capabilities"] = capabilities
+
+    def run(self) -> None:
+        self._task_suite.run(
+            self._predictor, capabilities=self._capabilities, max_examples=self._max_examples
+        )
+
+        # We pass in an IO object.
+        output_file = self._output_file or sys.stdout
+        self._task_suite.summary(file=output_file, **self._print_summary_args)
+
+        # If `_output_file` was None, there would be nothing to close.
+        if self._output_file is not None:
+            self._output_file.close()
+
+
+def _run_suite(args: argparse.Namespace) -> None:
+
+    task_suite = _get_task_suite(args)
+    predictor = _get_predictor(args)
+
+    print_summary_args = args.print_summary_args.strip()
+    if len(print_summary_args) <= 0:
+        print_summary_args = {}
+    else:
+        print_summary_args = json.loads(print_summary_args)
+
+    capabilities = args.capabilities
+    max_examples = args.max_examples
+
+    manager = _CheckListManager(
+        task_suite,
+        predictor,
+        capabilities,
+        max_examples,
+        args.output_file,
+        print_summary_args,
+    )
+    manager.run()