Con duct ls

README.md

@@ -139,12 +139,14 @@ usage: con-duct <command> [options]
 A suite of commands to manage or manipulate con-duct logs.
 
 positional arguments:
-  {pp,plot}   Available subcommands
-    pp        Pretty print a JSON log.
-    plot      Plot resource usage for an execution.
+  {pp,plot,ls}  Available subcommands
+    pp          Pretty print a JSON log.
+    plot        Plot resource usage for an execution.
+    ls          Print execution information for all runs matching
+                DUCT_OUTPUT_PREFIX.
 
 options:
-  -h, --help  show this help message and exit
+  -h, --help    show this help message and exit
 
 ```
 <!-- END EXTRAS HELP -->

setup.cfg

@@ -58,6 +58,7 @@ where = src
 [options.extras_require]
 all =
     matplotlib
+    PyYAML
 
 
 [options.entry_points]

src/con_duct/__main__.py

@@ -28,6 +28,9 @@
 lgr = logging.getLogger("con-duct")
 DEFAULT_LOG_LEVEL = os.environ.get("DUCT_LOG_LEVEL", "INFO").upper()
 
+DUCT_OUTPUT_PREFIX = os.getenv(
+    "DUCT_OUTPUT_PREFIX", ".duct/logs/{datetime_filesafe}-{pid}_"
+)
 ENV_PREFIXES = ("PBS_", "SLURM_", "OSG")
 SUFFIXES = {
     "stdout": "stdout",
@@ -712,9 +715,7 @@ def from_argv(
             "-p",
             "--output-prefix",
             type=str,
-            default=os.getenv(
-                "DUCT_OUTPUT_PREFIX", ".duct/logs/{datetime_filesafe}-{pid}_"
-            ),
+            default=DUCT_OUTPUT_PREFIX,
             help="File string format to be used as a prefix for the files -- the captured "
             "stdout and stderr and the resource usage logs. The understood variables are "
             "{datetime}, {datetime_filesafe}, and {pid}. "

src/con_duct/suite/ls.py

@@ -0,0 +1,91 @@
+import argparse
+import glob
+import json
+from typing import List
+import pyout
+import yaml
+from con_duct.__main__ import DUCT_OUTPUT_PREFIX, SummaryFormatter
+
+LS_SUMMARY_FORMAT = (
+    "Command: {command}\n"
+    "\tWall Clock Time: {execution_summary[wall_clock_time]:.3f} sec\n"
+    "\tMemory Peak Usage (RSS): {execution_summary[peak_rss]!S}\n"
+    "\tVirtual Memory Peak Usage (VSZ): {execution_summary[peak_vsz]!S}\n"
+    "\tMemory Peak Percentage: {execution_summary[peak_pmem]:.2f!N}%\n"
+    "\tCPU Peak Usage: {execution_summary[peak_pcpu]:.2f!N}%\n"
+)
+
+
+def load_duct_runs(info_files: List[str]) -> List[dict]:
+    loaded = []
+    for info_file in info_files:
+        with open(info_file) as file:
+            loaded.append(json.load(file))
+    return loaded
+
+
+def _flatten_dict(d, parent_key="", sep="."):
+    """Flatten a nested dictionary, creating keys as dot-separated paths."""
+    items = []
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.extend(_flatten_dict(v, new_key, sep=sep).items())
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
+def _restrict_row(include_only, row):
+    restricted = {}
+    for k, v in row.items():
+        if k in include_only:
+            restricted[k.split(".")[-1]] = v
+    return restricted
+
+
+def pyout_ls(run_data_list):
+    # Generate Tabular table to output
+    table = pyout.Tabular(
+        style=dict(
+            header_=dict(bold=True, transform=str.upper),
+            # Default styling could be provided from some collection of styling files
+            default_=dict(
+                color=dict(
+                    lookup={
+                        "Trix": "green",
+                        "110": "red",
+                        "100": "green",  # since no grey for now
+                    }
+                )
+            ),
+        ),
+    )
+    include_only = ["command", "execution_summary.exit_code"]
+    for row in run_data_list:
+        # table(row)
+        flattened = _flatten_dict(row)
+        table(_restrict_row(include_only, flattened))
+
+
+def ls(args: argparse.Namespace) -> int:
+    pattern = f"{DUCT_OUTPUT_PREFIX[:DUCT_OUTPUT_PREFIX.index('{')]}*_info.json"
+    info_files = glob.glob(pattern)
+    run_data_list = load_duct_runs(info_files)
+    if args.format == "summaries":
+        formatter = SummaryFormatter()  # TODO enable_colors=self.colors)
+        for data in run_data_list:
+            print(formatter.format(LS_SUMMARY_FORMAT, **data))
+    elif args.format == "pyout":
+        pyout_ls(run_data_list)
+    elif args.format == "json":
+        print(json.dumps(run_data_list))
+    elif args.format == "json_pp":
+        print(json.dumps(run_data_list, indent=True))
+    elif args.format == "yaml":
+        print(yaml.dump(run_data_list, default_flow_style=False))
+    else:
+        raise RuntimeError(
+            f"Unexpected format encountered: {args.format}. This should have been caught by argparse.",
+        )
+    return 0

src/con_duct/suite/main.py

@@ -1,6 +1,7 @@
 import argparse
 import sys
 from typing import List, Optional
+from con_duct.suite.ls import ls
 from con_duct.suite.plot import matplotlib_plot
 from con_duct.suite.pprint_json import pprint_json
 
@@ -46,6 +47,19 @@ def main(argv: Optional[List[str]] = None) -> None:
     # )
     parser_plot.set_defaults(func=matplotlib_plot)
 
+    parser_ls = subparsers.add_parser(
+        "ls",
+        help="Print execution information for all runs matching DUCT_OUTPUT_PREFIX.",
+    )
+    parser_ls.add_argument(
+        "-f",
+        "--format",
+        choices=("summaries", "json", "json_pp", "yaml", "pyout"),
+        default="summaries",  # TODO dry
+        help="Output format. TODO Fixme",
+    )
+    parser_ls.set_defaults(func=ls)
+
     args = parser.parse_args(argv)
 
     if args.command is None:

tox.ini

@@ -26,6 +26,7 @@ commands =
 deps =
     mypy
     data-science-types  # TODO replace archived, https://github.com/wearepal/data-science-types
+    types-PyYAML
     {[testenv]deps}
 commands =
     mypy src test