Merge pull request #1047 from dandi/gh-1046

Don't ingest a Zarr if nothing changed while uploading
dandi · Jul 6, 2022 · 9ce5b1e · 9ce5b1e
2 parents 8b28039 + 95cedc9
commit 9ce5b1e
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 9 deletions.
diff --git a/dandi/files.py b/dandi/files.py
@@ -961,6 +961,7 @@ def mkzarr() -> str:
         yield {"status": "initiating upload", "size": total_size}
         lgr.debug("%s: Beginning upload", asset_path)
         bytes_uploaded = 0
+        need_ingest = False
         upload_data = (
             zarr_id,
             client.get_url(f"/zarr/{zarr_id}/upload"),
@@ -991,6 +992,7 @@ def mkzarr() -> str:
                         )
                         for upspec in r
                     ]
+                    need_ingest = True
                     for fut in as_completed(futures):
                         try:
                             size = fut.result()
@@ -1024,15 +1026,19 @@ def mkzarr() -> str:
                 pluralize(len(old_zarr_files), "file"),
             )
             a.rmfiles(old_zarr_files, reingest=False)
-        lgr.debug("%s: Waiting for server to calculate Zarr checksum", asset_path)
-        yield {"status": "server calculating checksum"}
-        client.post(f"/zarr/{zarr_id}/ingest/")
-        while True:
-            sleep(2)
-            r = client.get(f"/zarr/{zarr_id}/")
-            if r["status"] == "Complete":
-                break
-        lgr.info("%s: Asset successfully uploaded", asset_path)
+            need_ingest = True
+        if need_ingest:
+            lgr.debug("%s: Waiting for server to calculate Zarr checksum", asset_path)
+            yield {"status": "server calculating checksum"}
+            client.post(f"/zarr/{zarr_id}/ingest/")
+            while True:
+                sleep(2)
+                r = client.get(f"/zarr/{zarr_id}/")
+                if r["status"] == "Complete":
+                    break
+            lgr.info("%s: Asset successfully uploaded", asset_path)
+        else:
+            lgr.info("%s: No changes made to Zarr", asset_path)
         yield {"status": "done", "asset": a}
 
 

diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py
@@ -219,6 +219,8 @@ def test_upload_zarr(new_dandiset: SampleDandiset) -> None:
     assert isinstance(asset, RemoteZarrAsset)
     assert asset.asset_type is AssetType.ZARR
     assert asset.path == "sample.zarr"
+    # Test that uploading again without any changes works:
+    new_dandiset.upload()
 
 
 def test_upload_different_zarr(tmp_path: Path, zarr_dandiset: SampleDandiset) -> None: