From 4d444607ec267aa5981c52df7e43866ada5036e0 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 19 Nov 2024 11:04:49 -0500 Subject: [PATCH 1/2] Fix parsing of manifest dates from S3 --- src/s3/streams.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/s3/streams.rs b/src/s3/streams.rs index d0f1b7f..1ea03ab 100644 --- a/src/s3/streams.rs +++ b/src/s3/streams.rs @@ -72,7 +72,14 @@ impl Stream for ListManifestDates { .common_prefixes .unwrap_or_default() .into_iter() - .filter_map(|CommonPrefix { prefix, .. }| prefix?.parse::().ok()) + .filter_map(|CommonPrefix { prefix, .. }| { + prefix? + .strip_suffix('/')? + .rsplit_once('/') + .map(|(_, s)| s)? + .parse::() + .ok() + }) .collect::>(); } } From 0252339ceec8527ca86ca66d86cb98dc8d3828e1 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 19 Nov 2024 11:05:35 -0500 Subject: [PATCH 2/2] Remove redundant `.csv.gz` extension from downloaded inventory lists --- src/s3/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/s3/mod.rs b/src/s3/mod.rs index bb00f0c..c1a43a8 100644 --- a/src/s3/mod.rs +++ b/src/s3/mod.rs @@ -164,8 +164,7 @@ impl S3Client { .rsplit_once('/') .map_or(&*fspec.key, |(_, after)| after); let url = self.inventory_base.with_key(&fspec.key); - let outfile = - self.make_dl_tempfile(&PathBuf::from(format!("data/{fname}.csv.gz")), &url)?; + let outfile = self.make_dl_tempfile(&PathBuf::from(format!("data/{fname}")), &url)?; self.download_object(&url, Some(&fspec.md5_checksum), &outfile) .await?; Ok(InventoryList::from_gzip_csv_file(url, outfile))