From 53f075dc58a13b54db16eb37bea07f56c441833a Mon Sep 17 00:00:00 2001 From: roytev Date: Tue, 19 Mar 2024 10:25:54 +0200 Subject: [PATCH 1/5] feat(output): save list of deleted artifacts as json --- README.md | 3 +++ artifactory_cleanup/artifactorycleanup.py | 11 ++++++++--- artifactory_cleanup/cli.py | 23 ++++++++++++----------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index acce4b6..a5636d3 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,9 @@ artifactory-cleanup --output=myfile.txt # Save the summary in a Json file artifactory-cleanup --output=myfile.txt --output-format=json + +# Save the summary in a json file and append a list with removed artifacts +artifactory-cleanup --output=myfile.json --output-format json-with-artifact-list ``` # Rules diff --git a/artifactory_cleanup/artifactorycleanup.py b/artifactory_cleanup/artifactorycleanup.py index af447b7..2a880ec 100644 --- a/artifactory_cleanup/artifactorycleanup.py +++ b/artifactory_cleanup/artifactorycleanup.py @@ -1,6 +1,6 @@ from concurrent.futures import ThreadPoolExecutor from datetime import date -from typing import List, Iterator +from typing import List, Iterator, Optional from attr import dataclass from requests import Session @@ -14,6 +14,7 @@ class CleanupSummary: policy_name: str artifacts_removed: int artifacts_size: int + removed_artifacts_list: Optional[dict] = None class ArtifactoryCleanup: @@ -25,12 +26,14 @@ def __init__( today: date, ignore_not_found: bool, worker_count: int, + output_format: str, ): self.session = session self.policies = policies self.destroy = destroy self.ignore_not_found = ignore_not_found self.worker_count = worker_count + self.output_format = output_format self._init_policies(today) @@ -72,12 +75,14 @@ def _delete(artifact): try: artifacts_size = sum([x["size"] for x in artifacts_to_remove]) print("Summary size: {}".format(artifacts_size)) - yield CleanupSummary( + summary = CleanupSummary( policy_name=policy.name, artifacts_size=artifacts_size, artifacts_removed=len(artifacts_to_remove), ) - + if self.output_format == "json-with-artifact-list": + summary.removed_artifacts_list = artifacts_to_remove + yield summary except KeyError: print("Summary size not defined") yield None diff --git a/artifactory_cleanup/cli.py b/artifactory_cleanup/cli.py index 5ad6f3a..05b285b 100644 --- a/artifactory_cleanup/cli.py +++ b/artifactory_cleanup/cli.py @@ -14,12 +14,12 @@ ArtifactoryCleanup, ) from artifactory_cleanup.base_url_session import BaseUrlSession +from artifactory_cleanup.context_managers import get_context_managers from artifactory_cleanup.errors import InvalidConfigError from artifactory_cleanup.loaders import ( PythonLoader, YamlConfigLoader, ) -from artifactory_cleanup.context_managers import get_context_managers requests.packages.urllib3.disable_warnings() @@ -88,7 +88,7 @@ class ArtifactoryCleanupCLI(cli.Application): _output_format = cli.SwitchAttr( "--output-format", - Set("table", "json", case_sensitive=False), + Set("table", "json", "json-with-artifact-list", case_sensitive=False), help="Choose the output format", default="table", requires=["--output"], @@ -140,7 +140,7 @@ def _create_output_file(self, result, filename, format): text = None if format == "table": text = self._format_table(result).get_string() - else: + elif format == "json" or format == "json-with-artifact-list": text = json.dumps(result) with open(filename, "w") as file: @@ -172,6 +172,7 @@ def main(self): today=today, ignore_not_found=self._ignore_not_found, worker_count=self._worker_count, + output_format=self._output_format, ) # Filter policies by name @@ -189,14 +190,14 @@ def main(self): continue total_size += summary.artifacts_size - result["policies"].append( - { - "name": summary.policy_name, - "file_count": summary.artifacts_removed, - "size": summary.artifacts_size, - } - ) - + policy = { + "name": summary.policy_name, + "file_count": summary.artifacts_removed, + "size": summary.artifacts_size + } + if summary.removed_artifacts_list is not None: + policy["artifacts_list"] = summary.removed_artifacts_list + result["policies"].append(policy) result["total_size"] = total_size self._print_table(result) From ad796af60a8197c072d06e065e2da76a32796c1e Mon Sep 17 00:00:00 2001 From: roytev Date: Tue, 19 Mar 2024 10:25:54 +0200 Subject: [PATCH 2/5] fixup! feat(output): save list of deleted artifacts as json --- README.md | 2 +- artifactory_cleanup/artifactorycleanup.py | 10 ++++++---- artifactory_cleanup/cli.py | 11 +++++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a5636d3..9e6e6ba 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ artifactory-cleanup --output=myfile.txt artifactory-cleanup --output=myfile.txt --output-format=json # Save the summary in a json file and append a list with removed artifacts -artifactory-cleanup --output=myfile.json --output-format json-with-artifact-list +artifactory-cleanup --output=myfile.json --output-format json --output-artifacts ``` # Rules diff --git a/artifactory_cleanup/artifactorycleanup.py b/artifactory_cleanup/artifactorycleanup.py index 2a880ec..a1d46dd 100644 --- a/artifactory_cleanup/artifactorycleanup.py +++ b/artifactory_cleanup/artifactorycleanup.py @@ -6,7 +6,7 @@ from requests import Session from artifactory_cleanup.errors import ArtifactoryCleanupException -from artifactory_cleanup.rules.base import CleanupPolicy, ArtifactDict +from artifactory_cleanup.rules.base import ArtifactsList, CleanupPolicy, ArtifactDict @dataclass @@ -14,7 +14,7 @@ class CleanupSummary: policy_name: str artifacts_removed: int artifacts_size: int - removed_artifacts_list: Optional[dict] = None + removed_artifacts_list: Optional[ArtifactsList] = None class ArtifactoryCleanup: @@ -27,6 +27,7 @@ def __init__( ignore_not_found: bool, worker_count: int, output_format: str, + output_artifacts: bool, ): self.session = session self.policies = policies @@ -34,6 +35,7 @@ def __init__( self.ignore_not_found = ignore_not_found self.worker_count = worker_count self.output_format = output_format + self.output_artifacts = output_artifacts, self._init_policies(today) @@ -41,7 +43,7 @@ def _init_policies(self, today): for policy in self.policies: policy.init(self.session, today) - def cleanup(self, block_ctx_mgr, test_ctx_mgr) -> Iterator[CleanupSummary]: + def cleanup(self, block_ctx_mgr, test_ctx_mgr) -> Iterator[Optional[CleanupSummary]]: for policy in self.policies: with block_ctx_mgr(policy.name): # Prepare @@ -80,7 +82,7 @@ def _delete(artifact): artifacts_size=artifacts_size, artifacts_removed=len(artifacts_to_remove), ) - if self.output_format == "json-with-artifact-list": + if self.output_format == "json" and self.output_artifacts: summary.removed_artifacts_list = artifacts_to_remove yield summary except KeyError: diff --git a/artifactory_cleanup/cli.py b/artifactory_cleanup/cli.py index 05b285b..719cc72 100644 --- a/artifactory_cleanup/cli.py +++ b/artifactory_cleanup/cli.py @@ -88,13 +88,19 @@ class ArtifactoryCleanupCLI(cli.Application): _output_format = cli.SwitchAttr( "--output-format", - Set("table", "json", "json-with-artifact-list", case_sensitive=False), + Set("table", "json", case_sensitive=False), help="Choose the output format", default="table", requires=["--output"], mandatory=False, ) + _output_artifacts = cli.Flag( + "--output-artifacts", help="When --output-format is json, append the list of all deleted artifacts to --output.", + mandatory=False, + default=False, + ) + @property def VERSION(self): # To prevent circular imports @@ -140,7 +146,7 @@ def _create_output_file(self, result, filename, format): text = None if format == "table": text = self._format_table(result).get_string() - elif format == "json" or format == "json-with-artifact-list": + elif format == "json": text = json.dumps(result) with open(filename, "w") as file: @@ -173,6 +179,7 @@ def main(self): ignore_not_found=self._ignore_not_found, worker_count=self._worker_count, output_format=self._output_format, + output_artifacts=self._output_artifacts, ) # Filter policies by name From 8d4e2aa55de493b888c2325fe2daab31b33d2fba Mon Sep 17 00:00:00 2001 From: kygr Date: Mon, 19 Aug 2024 10:26:04 +0200 Subject: [PATCH 3/5] fixup! feat(output): save list of deleted artifacts as json --- README.md | 4 ++-- artifactory_cleanup/artifactorycleanup.py | 9 ++------- artifactory_cleanup/cli.py | 15 +++++++-------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 9e6e6ba..37d3f85 100644 --- a/README.md +++ b/README.md @@ -153,10 +153,10 @@ docker run -v "$(pwd)":/app devopshq/artifactory-cleanup artifactory-cleanup --l # Save the table summary in a file artifactory-cleanup --output=myfile.txt -# Save the summary in a Json file +# Save the summary in a json file artifactory-cleanup --output=myfile.txt --output-format=json -# Save the summary in a json file and append a list with removed artifacts +# Save the summary in a json file and append the list of all removed artifacts artifactory-cleanup --output=myfile.json --output-format json --output-artifacts ``` diff --git a/artifactory_cleanup/artifactorycleanup.py b/artifactory_cleanup/artifactorycleanup.py index a1d46dd..873b092 100644 --- a/artifactory_cleanup/artifactorycleanup.py +++ b/artifactory_cleanup/artifactorycleanup.py @@ -14,7 +14,7 @@ class CleanupSummary: policy_name: str artifacts_removed: int artifacts_size: int - removed_artifacts_list: Optional[ArtifactsList] = None + removed_artifacts: ArtifactsList class ArtifactoryCleanup: @@ -26,16 +26,12 @@ def __init__( today: date, ignore_not_found: bool, worker_count: int, - output_format: str, - output_artifacts: bool, ): self.session = session self.policies = policies self.destroy = destroy self.ignore_not_found = ignore_not_found self.worker_count = worker_count - self.output_format = output_format - self.output_artifacts = output_artifacts, self._init_policies(today) @@ -81,9 +77,8 @@ def _delete(artifact): policy_name=policy.name, artifacts_size=artifacts_size, artifacts_removed=len(artifacts_to_remove), + artifacts=artifacts_to_remove ) - if self.output_format == "json" and self.output_artifacts: - summary.removed_artifacts_list = artifacts_to_remove yield summary except KeyError: print("Summary size not defined") diff --git a/artifactory_cleanup/cli.py b/artifactory_cleanup/cli.py index 719cc72..1914e35 100644 --- a/artifactory_cleanup/cli.py +++ b/artifactory_cleanup/cli.py @@ -96,7 +96,8 @@ class ArtifactoryCleanupCLI(cli.Application): ) _output_artifacts = cli.Flag( - "--output-artifacts", help="When --output-format is json, append the list of all deleted artifacts to --output.", + "--output-artifacts", + help="When --output-format is json, append the list of all removed artifacts to output", mandatory=False, default=False, ) @@ -143,13 +144,13 @@ def _print_table(self, result: dict): print(self._format_table(result)) def _create_output_file(self, result, filename, format): - text = None + text = "" if format == "table": text = self._format_table(result).get_string() elif format == "json": - text = json.dumps(result) + text = json.dumps(result, indent=4) - with open(filename, "w") as file: + with open(filename, "w", encoding="utf-8") as file: file.write(text) def main(self): @@ -178,8 +179,6 @@ def main(self): today=today, ignore_not_found=self._ignore_not_found, worker_count=self._worker_count, - output_format=self._output_format, - output_artifacts=self._output_artifacts, ) # Filter policies by name @@ -202,8 +201,8 @@ def main(self): "file_count": summary.artifacts_removed, "size": summary.artifacts_size } - if summary.removed_artifacts_list is not None: - policy["artifacts_list"] = summary.removed_artifacts_list + if summary.removed_artifacts is not None and self._output_artifacts: + policy["removed_artifacts"] = summary.removed_artifacts result["policies"].append(policy) result["total_size"] = total_size From 9139967108c91d969d73bcb2c5461b952fcadd9e Mon Sep 17 00:00:00 2001 From: allburov Date: Tue, 20 Aug 2024 12:44:13 +0700 Subject: [PATCH 4/5] removed_artifacts can not be None --- artifactory_cleanup/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifactory_cleanup/cli.py b/artifactory_cleanup/cli.py index 1914e35..999744b 100644 --- a/artifactory_cleanup/cli.py +++ b/artifactory_cleanup/cli.py @@ -201,7 +201,7 @@ def main(self): "file_count": summary.artifacts_removed, "size": summary.artifacts_size } - if summary.removed_artifacts is not None and self._output_artifacts: + if self._output_artifacts: policy["removed_artifacts"] = summary.removed_artifacts result["policies"].append(policy) result["total_size"] = total_size From bdafcf89759bba982d377bbf71397d3259310951 Mon Sep 17 00:00:00 2001 From: allburov Date: Tue, 20 Aug 2024 12:45:42 +0700 Subject: [PATCH 5/5] removed_artifacts --- artifactory_cleanup/artifactorycleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifactory_cleanup/artifactorycleanup.py b/artifactory_cleanup/artifactorycleanup.py index 873b092..907fe70 100644 --- a/artifactory_cleanup/artifactorycleanup.py +++ b/artifactory_cleanup/artifactorycleanup.py @@ -77,7 +77,7 @@ def _delete(artifact): policy_name=policy.name, artifacts_size=artifacts_size, artifacts_removed=len(artifacts_to_remove), - artifacts=artifacts_to_remove + removed_artifacts=artifacts_to_remove ) yield summary except KeyError: