Merge pull request #234 from UtrechtUniversity/develop

v1.0.1

.github/workflows/integration-tests-yoda.yml

@@ -32,7 +32,7 @@ jobs:
 
     - name: Clone Yoda repo for Docker Setup
       run: |
-        git clone -b development --single-branch https://github.com/UtrechtUniversity/yoda.git
+        git clone -b release-1.9 --single-branch https://github.com/UtrechtUniversity/yoda.git
 
     - name: Prepare hosts file for integration tests
       run: |

README.md

@@ -1,5 +1,5 @@
 # iBridges <img src="docs/logo.png" width="150" align="right">
-iBridges is library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
+iBridges is a library for scientific programmers who are working with data in iRODS. We provide a wrapper around the [python-irodsclient](https://pypi.org/project/python-irodsclient/) to facilitate easy interaction with the iRODS server. iBridges is currently still in very active development.
 
 [![PyPI version](https://badge.fury.io/py/ibridges.svg)](https://badge.fury.io/py/ibridges)
 [![](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-irods.yml/badge.svg?branch=develop)](https://github.com/UtrechtUniversity/iBridges/actions/workflows/integration-tests-irods.yml) [![](https://github.com/UtrechtUniversity/iBridges/actions/workflows/main.yml/badge.svg?branch=develop)](https://github.com/UtrechtUniversity/iBridges/actions/workflows/main.yml) 
@@ -11,7 +11,7 @@ iBridges is library for scientific programmers who are working with data in iROD
 - Works on Windows, Mac OS and Linux
 - Runs on Python 3.8 or higher.
 - Supported iRODS server versions: 4.2.11 or higher and 4.3.0 or higher.
-- **Interactive connection** to your iRods server.
+- **Interactive connection** to your iRODS server.
 - **Upload** and **Download** your data.
 - Manipulate the **metadata** on the iRODS server.
 - **Synchronise** your data between your local computer and the iRODS server.

docker/irods_client/tests/test_cli.py

@@ -1,4 +1,5 @@
 import hashlib
+import json
 import subprocess
 from pathlib import Path
 
@@ -20,6 +21,7 @@ def pass_opts(config):
         pass_opts = {}
     else:
         pass_opts = {"input": config["password"].encode()}
+    pass_opts["check"] = True
     return pass_opts
 
 def _get_digest(obj_or_file):
@@ -47,27 +49,24 @@ def test_upload_download_cli(session, config, testdata, tmpdir, irods_env_file,
     # Test the upload function by uploading a single file.
     ipath = IrodsPath(session, "~", "plant.rtf")
     ipath.remove()
-    subprocess.run(["ibridges", "init", irods_env_file],
-                   check=True, **pass_opts)
+    subprocess.run(["ibridges", "init", irods_env_file], **pass_opts)
 
     if "resc2" in config["resources"]:
         subprocess.run(["ibridges", "upload", testdata/"plant.rtf", "irods:" + str(ipath),
-                        "--overwrite", "--resource", "resc2"],
-                        check=True, **pass_opts)
+                        "--overwrite", "--resource", "resc2"], **pass_opts)
     else:
         subprocess.run(["ibridges", "upload", testdata/"plant.rtf", "irods:" + str(ipath),
-                        "--overwrite"],
-                        check=True, **pass_opts)
+                        "--overwrite"], **pass_opts)
     assert ipath.dataobject_exists()
 
     # Download the same file and check if they are equal.
     assert isinstance(testdata, Path)
     if "resc2" in config["resources"]:
         subprocess.run(["ibridges", "download", "irods:~/plant.rtf", testdata/"plant2.rtf",
-                        "--resource", "resc2"], check=True, **pass_opts)
+                        "--resource", "resc2"], **pass_opts)
     else:
         subprocess.run(["ibridges", "download", "irods:~/plant.rtf", testdata/"plant2.rtf"],
-                        check=True, **pass_opts)
+                        **pass_opts)
     assert Path(testdata/"plant2.rtf").is_file()
 
     _check_files_equal(testdata/"plant2.rtf", testdata/"plant.rtf")
@@ -77,25 +76,57 @@ def test_upload_download_cli(session, config, testdata, tmpdir, irods_env_file,
     ipath = IrodsPath(session, "~", "test")
     ipath.remove()
     create_collection(session, ipath)
-    subprocess.run(["ibridges", "upload", testdata, "irods:~/test", "--overwrite"], check=True, **pass_opts)
+    subprocess.run(["ibridges", "upload", testdata, "irods:~/test", "--overwrite"], **pass_opts)
     for fname in testdata.glob("*"):
         assert ((ipath / "testdata" / fname.name).dataobject_exists()
                 or (ipath / "testdata" / fname.name).collection_exists())
 
     # Download the created collection and check whether the files are the same.
-    subprocess.run(["ibridges", "download", "irods:~/test/testdata", tmpdir], check=True, **pass_opts)
+    subprocess.run(["ibridges", "download", "irods:~/test/testdata", tmpdir], **pass_opts)
     for fname in testdata.glob("*"):
         assert _check_files_equal(testdata/fname.name, tmpdir/"testdata"/fname.name)
     Path(tmpdir/"testdata").unlink
 
     # Synchronize a collection to a temporary directory and check if they are the same.
-    subprocess.run(["ibridges", "sync", "irods:~/test/testdata", tmpdir], check=True, **pass_opts)
+    subprocess.run(["ibridges", "sync", "irods:~/test/testdata", tmpdir], **pass_opts)
     for fname in testdata.glob("*"):
         assert _check_files_equal(testdata/fname.name, tmpdir/fname.name)
 
+def test_upload_download_metadata(session, config, testdata, tmpdir, irods_env_file, pass_opts):
+    ipath_collection = IrodsPath(session, "meta_test")
+    ipath_collection.remove()
+    subprocess.run(["ibridges", "upload", testdata, f"irods:{ipath_collection}"],
+                   **pass_opts)
+    assert ipath_collection.exists()
+    meta = ipath_collection.meta
+    meta.add("some_key", "some_val")
+    subprocess.run(["ibridges", "download", f"irods:{ipath_collection}", tmpdir, "--metadata"],
+                   **pass_opts)
+    meta_fp = tmpdir / "meta_test" / ".ibridges_metadata.json"
+    assert meta_fp.isfile()
+    with open(meta_fp, "r", encoding="utf-8") as handle:
+        metadata = json.load(handle)
+        assert metadata["items"][0]["name"] == "meta_test"
+        assert metadata["items"][0]["metadata"][0][0] == "some_key"
+        assert metadata["items"][0]["metadata"][0][1] == "some_val"
+    ipath_collection.remove()
+
+    # Check uploading metadata with upload
+    subprocess.run(["ibridges", "upload", tmpdir / "meta_test", "irods:",
+                    "--metadata"], **pass_opts)
+    assert ("some_key", "some_val") in ipath_collection.meta
+
+    # Check uploading metadata with sync
+    ipath_collection.meta.delete("some_key", "some_val")
+    subprocess.run(["ibridges", "sync", tmpdir / "meta_test", "irods:meta_test",
+                    "--metadata"], **pass_opts)
+    assert ("some_key", "some_val") in ipath_collection.meta
+
+    ipath_collection.remove()
+
 
 def test_list_cli(config, pass_opts, irods_env_file):
     # Check if the listing works without any errors.
-    subprocess.run(["ibridges", "init", irods_env_file], check=True, **pass_opts)
-    subprocess.run(["ibridges", "list"], check=True, **pass_opts)
-    subprocess.run(["ibridges", "list", "irods:test"], check=True, **pass_opts)
+    subprocess.run(["ibridges", "init", irods_env_file], **pass_opts)
+    subprocess.run(["ibridges", "list"], **pass_opts)
+    subprocess.run(["ibridges", "list", "irods:test"], **pass_opts)

docker/irods_client/tests/test_data_ops.py

@@ -1,12 +1,10 @@
 import hashlib
+import json
 from pathlib import Path
 
 import pytest
 
-from ibridges.data_operations import (
-    download,
-    upload,
-)
+from ibridges.data_operations import apply_meta_archive, create_meta_archive, download, sync, upload
 from ibridges.path import IrodsPath
 from ibridges.util import is_collection, is_dataobject
 
@@ -29,7 +27,7 @@ def _check_files_equal(*files):
 def _check_count(ops, nlist):
     for var, count, in zip(["create_collection", "create_dir", "download", "upload"],
                           nlist):
-        assert len(ops[var]) == count
+        assert len(getattr(ops, var)) == count
 
 
 
@@ -46,20 +44,21 @@ def test_upload_download_dataset(session, testdata):
     ops = download(session, ipath, testdata/"plant.rtf.copy", overwrite=True)
     assert _check_files_equal(testdata/"plant.rtf.copy", testdata/"plant.rtf")
     _check_count(ops, [0, 0, 1, 0])
+    (testdata/"plant.rtf.copy").unlink()
 
 
 def test_upload_download_collection(session, testdata, tmpdir):
     ipath = IrodsPath(session, "~", "test")
     ipath.remove()
     ops = upload(session, testdata, ipath)
-    _check_count(ops, [2, 0, 0, 7])
+    _check_count(ops, [3, 0, 0, 6])
     collection = ipath.collection
     assert is_collection(collection)
     assert not is_dataobject(collection)
     with pytest.raises(ValueError):
         ipath.dataobject
     ops = download(session, ipath, tmpdir/"test")
-    _check_count(ops, [0, 4, 7, 0])
+    _check_count(ops, [0, 4, 6, 0])
     files = list(testdata.glob("*"))
 
     for cur_file in files:
@@ -71,3 +70,53 @@ def test_upload_download_collection(session, testdata, tmpdir):
             with open(copy_file, "r") as handle:
                 copy_data = handle.read()
             assert copy_data == orig_data
+
+
+def test_meta_archive(session, testdata, tmpdir):
+    ipath = IrodsPath(session, "test")
+    ipath.remove()
+    sync(session, testdata, ipath)
+    assert len(list(ipath.meta)) == 0
+    meta_list = [
+        (ipath, ("root", "true", None)),
+        (ipath / "more_data", ("more_data", "false", "kg")),
+        (ipath / "more_data" / "polarbear.txt", ("is_polar", "true", "bool")),
+    ]
+    for cur_ipath, meta_data in meta_list:
+        cur_ipath.meta.add(*meta_data)
+    meta_fp = tmpdir / "meta.json"
+    create_meta_archive(session, ipath, meta_fp)
+    with open(meta_fp, "r") as handle:
+        meta_dict = json.load(handle)
+
+    assert "ibridges_metadata_version" in meta_dict
+    assert meta_dict["recursive"] is True
+    assert meta_dict["root_path"] == str(ipath)
+    assert len(meta_dict["items"]) == 8
+
+    def _find_meta_dict(abs_path):
+        rel_path = abs_path.relative_to(ipath)
+        for item in meta_dict["items"]:
+            if item["rel_path"] == str(rel_path):
+                return item
+        raise ValueError("Cannot find item in dictionary.")
+
+    def _check_in_metadict(expected_metadata, retrieved_metadata):
+        for meta_item in retrieved_metadata:
+            if all(meta_item[i] == expected_metadata[i] for i in range(len(expected_metadata))):
+                return True
+        return False
+
+    # Check if the metadata is in the file, then delete it remotely
+    for cur_ipath, meta_data in meta_list:
+        print(cur_ipath)
+        cur_meta_dict = _find_meta_dict(cur_ipath)
+        assert _check_in_metadict(meta_data, cur_meta_dict["metadata"])
+        # assert meta_data in cur_meta_dict["metadata"]
+        cur_ipath.meta.delete(meta_data[0], meta_data[1])
+
+    # Apply the archive and see if it has arrived.
+    apply_meta_archive(session, meta_fp, ipath)
+
+    for cur_ipath, meta_data in meta_list:
+        assert meta_data in cur_ipath.meta

docker/irods_client/tests/test_meta.py

@@ -1,9 +1,12 @@
+import json
+
 import irods
 import pytest
 from pytest import mark
 
-from ibridges.export_metadata import export_metadata_to_dict
+from ibridges.data_operations import Operations
 from ibridges.meta import MetaData
+from ibridges.path import IrodsPath
 
 
 @mark.parametrize("item_name", ["collection", "dataobject"])
@@ -13,11 +16,11 @@ def test_meta(item_name, request):
     meta.clear()
 
     assert len(str(meta)) == 0
-    assert len(list(meta)) == 0
+    assert len(meta) == 0
 
     # Add key, value pair
     meta.add("x", "y")
-    assert len(list(meta)) == 1
+    assert len(meta) == 1
     assert list(meta)[0].name == "x"
     assert list(meta)[0].value == "y"
     assert list(meta)[0].units is None
@@ -29,13 +32,13 @@ def test_meta(item_name, request):
 
     # Same key, but different value
     meta.add("x", "z")
-    assert len(list(meta)) == 2
-    assert len(str(meta).split("\n")) == 3  #\n at the end
+    assert len(meta) == 2
+    assert len(str(meta).split("\n")) == 2
     assert ("x", "z") in meta
 
     # Same key, value different units
     meta.add("x", "z", "m")
-    assert len(list(meta)) == 3
+    assert len(meta) == 3
     assert ("x", "z", "m") in meta
 
     # Test that we cannot add the same metadata twice
@@ -49,13 +52,13 @@ def test_meta(item_name, request):
     with pytest.raises(KeyError):
         meta.delete("x", "z", "kg")
     meta.delete("x", "z", "m")
-    assert len(list(meta)) == 2
+    assert len(meta) == 2
 
     meta.delete("x", "z")
-    assert len(list(meta)) == 1
+    assert len(meta) == 1
 
-    meta.delete("x", None)
-    assert len(list(meta)) == 0
+    meta.delete("x")
+    assert len(meta) == 0
 
     meta.add("x", "y")
     meta.add("y", "z")
@@ -90,7 +93,18 @@ def test_metadata_todict(item_name, request):
         assert value in test_dict.values()
 
 @mark.parametrize("item_name", ["collection", "dataobject"])
-def test_metadata_export(item_name, request, session):
+def test_metadata_export(item_name, request, session, tmpdir):
+    tmp_file = tmpdir/"meta.json"
     item = request.getfixturevalue(item_name)
-    res = export_metadata_to_dict(MetaData(item), session)
-    assert isinstance(res, dict)
+    meta_dict = MetaData(item).to_dict()
+    assert isinstance(meta_dict, dict)
+    assert "name" in meta_dict
+    assert "irods_id" in meta_dict
+    assert "metadata" in meta_dict
+
+    ops = Operations()
+    ops.add_meta_download(IrodsPath(session, item.path), IrodsPath(session, item.path), tmp_file)
+    ops.execute(session)
+    with open(tmp_file, "r", encoding="utf-8"):
+        new_meta_dict = json.load(tmp_file)
+    assert isinstance(new_meta_dict, dict)

docker/irods_client/tests/test_sync.py

@@ -1,5 +1,6 @@
-from ibridges.data_operations import _calc_checksum, create_collection, sync
+from ibridges.data_operations import create_collection, sync
 from ibridges.path import IrodsPath
+from ibridges.util import calc_checksum
 
 
 def test_sync_dry_run(session, testdata, capsys):
@@ -16,10 +17,10 @@ def test_sync_dry_run(session, testdata, capsys):
                dry_run=True,
                copy_empty_folders=True)
 
-    assert len(ops["create_collection"]) == 1
-    assert len(ops["create_dir"]) == 0
-    assert len(ops["download"]) == 0
-    assert len(ops["upload"]) == 7
+    assert len(ops.create_collection) == 1
+    assert len(ops.create_dir) == 0
+    assert len(ops.download) == 0
+    assert len(ops.upload) == 6
     assert len(coll.data_objects)+len(coll.subcollections)==0, "Dry run did sync"
 
 
@@ -42,7 +43,7 @@ def test_sync_upload_download(session, testdata, tmpdir):
         if cur_file.is_file():
             assert s_ipath.dataobject_exists(), "File not uploaded"
             cur_obj=s_ipath.dataobject
-            assert _calc_checksum(cur_file)==(cur_obj.checksum
+            assert calc_checksum(cur_file)==(cur_obj.checksum
                 if len(cur_obj.checksum)>0
                 else cur_obj.chksum()), "Checksums not identical after upload"
 
@@ -51,7 +52,7 @@ def test_sync_upload_download(session, testdata, tmpdir):
     s_ipath = IrodsPath(session, "~/empty/more_data/polarbear.txt")
     assert s_ipath.dataobject_exists(), "File in subfolder not uploaded"
     obj = s_ipath.dataobject
-    assert _calc_checksum(testdata / "more_data" / "polarbear.txt")== \
+    assert calc_checksum(testdata / "more_data" / "polarbear.txt")== \
         (obj.checksum if len(obj.checksum)>0 else obj.chksum()), \
             "Checksums not identical after upload"
 
@@ -66,12 +67,12 @@ def test_sync_upload_download(session, testdata, tmpdir):
     for cur_file in list(testdata.glob("*")):
         if cur_file.is_file():
             assert (tmpdir / cur_file.name).exists(), "File not downloaded"
-            assert _calc_checksum(tmpdir / cur_file.name)== \
-                _calc_checksum(testdata / cur_file.name), "Checksums not identical after download"
+            assert calc_checksum(tmpdir / cur_file.name)== \
+                calc_checksum(testdata / cur_file.name), "Checksums not identical after download"
         elif cur_file.is_dir():
             assert (tmpdir / cur_file.name).exists(), "Subfolder not downloaded"
 
     assert (tmpdir / "more_data" / "polarbear.txt").exists(), "File in subfolder not downloaded"
-    assert _calc_checksum(tmpdir / "more_data" / "polarbear.txt")== \
-        _calc_checksum(testdata  / "more_data" / "polarbear.txt"), \
+    assert calc_checksum(tmpdir / "more_data" / "polarbear.txt")== \
+        calc_checksum(testdata  / "more_data" / "polarbear.txt"), \
             "Checksums not identical after download"