Merge pull request #256 from UtrechtUniversity/develop

V1.1.0

docker/irods_catalog_provider/entrypoint.sh

@@ -36,4 +36,6 @@ su irods -c 'bash -c "./irodsctl stop"'
 
 echo "Starting server"
 cd /usr/sbin
+#sed 's/"irods_default_hash_scheme": "SHA256",//g' /var/lib/irods/.irods/irods_environment.json > t.json
+#mv t.json /var/lib/irods/.irods/irods_environment.json
 su irods -c 'bash -c "./irodsServer -u"'

docker/irods_client/tests/conftest.py

@@ -39,7 +39,6 @@ def config(config_dir):
 @pytest.fixture(scope="session")
 def session(irods_env, config):
     session = Session(irods_env=irods_env, password=config["password"])
-    ipath = IrodsPath(session, "~")
     yield session
     del session
 
@@ -51,7 +50,9 @@ def testdata():
 
 @pytest.fixture(scope="session")
 def collection(session):
-    return create_collection(session, IrodsPath(session, "~", "test_collection"))
+    coll = create_collection(session, IrodsPath(session, "~", "test_collection"))
+    yield coll
+    IrodsPath(session, coll.path).remove()
 
 
 @pytest.fixture(scope="session")

docker/irods_client/tests/test_cli.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 
 import pytest
+from pytest import mark
 
 from ibridges import IrodsPath
 from ibridges.data_operations import create_collection
@@ -20,7 +21,8 @@ def pass_opts(config):
             or config.get("has_cached_pw", False)):
         pass_opts = {}
     else:
-        pass_opts = {"input": config["password"].encode()}
+        pass_opts = {"input": config["password"]}
+    pass_opts["text"] = True
     pass_opts["check"] = True
     return pass_opts
 
@@ -71,6 +73,7 @@ def test_upload_download_cli(session, config, testdata, tmpdir, irods_env_file,
 
     _check_files_equal(testdata/"plant2.rtf", testdata/"plant.rtf")
     (testdata/"plant2.rtf").unlink()
+    ipath.remove()
 
     # Upload a directory and check if the dataobjects and collections are created.
     ipath = IrodsPath(session, "~", "test")
@@ -91,6 +94,8 @@ def test_upload_download_cli(session, config, testdata, tmpdir, irods_env_file,
     subprocess.run(["ibridges", "sync", "irods:~/test/testdata", tmpdir], **pass_opts)
     for fname in testdata.glob("*"):
         assert _check_files_equal(testdata/fname.name, tmpdir/fname.name)
+    ipath.remove()
+
 
 def test_upload_download_metadata(session, config, testdata, tmpdir, irods_env_file, pass_opts):
     ipath_collection = IrodsPath(session, "meta_test")
@@ -125,8 +130,47 @@ def test_upload_download_metadata(session, config, testdata, tmpdir, irods_env_f
     ipath_collection.remove()
 
 
-def test_list_cli(config, pass_opts, irods_env_file):
+def test_list_cli(config, pass_opts, irods_env_file, collection):
     # Check if the listing works without any errors.
     subprocess.run(["ibridges", "init", irods_env_file], **pass_opts)
     subprocess.run(["ibridges", "list"], **pass_opts)
-    subprocess.run(["ibridges", "list", "irods:test"], **pass_opts)
+    subprocess.run(["ibridges", "list", f"irods:{collection.path}"], **pass_opts)
+
+
+@mark.parametrize(
+    "search,nlines",
+    [
+        (["--path-pattern", "test_search"], 1),
+        (["--path-pattern", "test_search2"], 0),
+        (["--path-pattern", "test_search/%"], 6),
+        (["--path-pattern", "test_search/%/%"], 1),
+        (["--path-pattern", "%.rtf"], 4),
+        (["--checksum", "sha2:uJzC+gqi59rVJu8PoBAaTstNUUnFMxW9HsJzsJUb1ao="], 1),
+        (["--checksum", "sha2:uJzC+gqi5%"], 1),
+        (["--path-pattern", "%", "--item_type", "data_object"], 6),
+        (["--path-pattern", "%", "--item_type", "collection"], 2),
+        (["--path-pattern", "%/%", "--item_type", "collection"], 2),
+        (["--metadata", "search"], 1),
+        (["--metadata", "search", "sval", "kg"], 1),
+        (["--metadata", "search", "--metadata", "search2"], 1),
+        (["--metadata", "search", "--metadata", "does not exist"], 0),
+    ]
+)
+def test_search_cli(session, config, pass_opts, irods_env_file, testdata, search, nlines):
+    subprocess.run(["ibridges", "init", irods_env_file], **pass_opts)
+    ipath_coll = IrodsPath(session, "test_search_x", "test_search")
+    IrodsPath.create_collection(session, ipath_coll)
+    subprocess.run(["ibridges", "sync", testdata, "irods:test_search_x/test_search"], **pass_opts)
+    assert ipath_coll.collection_exists()
+    ipath_coll.meta.clear()
+    ipath_coll.meta.add("search", "sval", "kg")
+    ipath_coll.meta.add("search2", "small")
+
+    ret = subprocess.run(["ibridges", "search", "irods:test_search_x", *search], capture_output=True,
+                         **pass_opts)
+    stripped_str = ret.stdout.strip("\n")
+    if stripped_str == "":
+        assert nlines == 0
+    else:
+        assert len([x for x in stripped_str.split("\n") if not x.startswith("Your iRODS")]) == nlines
+    ipath_coll.remove()

docker/irods_client/tests/test_data_ops.py

@@ -41,10 +41,38 @@ def test_upload_download_dataset(session, testdata):
     assert not is_collection(data_obj)
     with pytest.raises(ValueError):
         _ = ipath.collection
+
+    # Check the overwrite and ignore_err parameters
+    with pytest.raises(FileExistsError):
+        upload(session, testdata/"plant.rtf", IrodsPath(session))
+    ops = upload(session, testdata/"plant.rtf", IrodsPath(session), overwrite=True)
+    assert len(ops.upload) == 0
+    with ipath.open("w") as handle:
+        handle.write("test".encode())
+    ops = upload(session, testdata/"plant.rtf", ipath, overwrite=False, ignore_err=True)
+    assert len(ops.upload) == 0
+    ops = upload(session, testdata/"plant.rtf", ipath, overwrite=True, ignore_err=True)
+    assert len(ops.upload) == 1
+
+    # Test downloading it back
     ops = download(session, ipath, testdata/"plant.rtf.copy", overwrite=True)
     assert _check_files_equal(testdata/"plant.rtf.copy", testdata/"plant.rtf")
     _check_count(ops, [0, 0, 1, 0])
-    (testdata/"plant.rtf.copy").unlink()
+
+    # Check overwrite and ignore_err parameters
+    lpath = testdata/"plant.rtf.copy"
+    ops = download(session, ipath, lpath, overwrite=True)
+    assert len(ops.download) == 0
+    with pytest.raises(FileExistsError):
+        download(session, ipath, lpath)
+    ops = download(session, ipath, lpath, overwrite=False, ignore_err=True)
+    assert len(ops.download) == 0
+    with ipath.open("w") as handle:
+        handle.write("test".encode())
+    ops = download(session, ipath, lpath, overwrite=True)
+    assert len(ops.download) == 1
+    ipath.remove()
+    lpath.unlink()
 
 
 def test_upload_download_collection(session, testdata, tmpdir):
@@ -57,6 +85,22 @@ def test_upload_download_collection(session, testdata, tmpdir):
     assert not is_dataobject(collection)
     with pytest.raises(ValueError):
         ipath.dataobject
+
+    # Check overwrite and ignore_err parameters
+    with pytest.raises(FileExistsError):
+        upload(session, testdata, ipath)
+    ops = upload(session, testdata, ipath, ignore_err=True)
+    _check_count(ops, [0, 0, 0, 0])
+    bunny_ipath = (ipath / "testdata" / "bunny.rtf")
+    bunny_ipath.remove()
+    ops = upload(session, testdata, ipath, overwrite=True)
+    _check_count(ops, [0, 0, 0, 1])
+    with bunny_ipath.open("w") as handle:
+        handle.write("est".encode())
+    ops = upload(session, testdata, ipath, overwrite=True)
+    _check_count(ops, [0, 0, 0, 1])
+
+    # Check if the downloaded collection is the same again.
     ops = download(session, ipath, tmpdir/"test")
     _check_count(ops, [0, 4, 6, 0])
     files = list(testdata.glob("*"))
@@ -71,6 +115,19 @@ def test_upload_download_collection(session, testdata, tmpdir):
                 copy_data = handle.read()
             assert copy_data == orig_data
 
+    # Check overwrite and ignore_err parameters
+    with pytest.raises(FileExistsError):
+        download(session, ipath, tmpdir/"test")
+    ops = download(session, ipath, tmpdir/"test", overwrite=True)
+    _check_count(ops, [0, 0, 0, 0])
+    with bunny_ipath.open("w") as handle:
+        handle.write("testxx".encode())
+    ops = download(session, ipath, tmpdir/"test", ignore_err=True)
+    _check_count(ops, [0, 0, 0, 0])
+    ops = download(session, ipath, tmpdir/"test", overwrite=True)
+    _check_count(ops, [0, 0, 1, 0])
+    ipath.remove()
+
 
 def test_meta_archive(session, testdata, tmpdir):
     ipath = IrodsPath(session, "test")

docker/irods_client/tests/test_path.py

@@ -0,0 +1,50 @@
+from ibridges.path import IrodsPath
+import pytest
+from irods.exception import DataObjectDoesNotExist
+
+
+def test_path_open_error(session, collection):
+    ipath = IrodsPath(session, "open_err_test.txt")
+    ipath.remove()
+    coll_ipath = IrodsPath(session, collection.path)
+
+    # Reading data objects that do no exist should raise an error.
+    with pytest.raises(DataObjectDoesNotExist):
+        with ipath.open("r") as handle:
+            handle.read()
+
+    with pytest.raises(DataObjectDoesNotExist):
+        with ipath.open("a") as handle:
+            handle.write("abc")
+
+    # We should not be able to open collections.
+    with pytest.raises(ValueError):
+        with coll_ipath.open("r") as handle:
+            handle.read()
+
+    with pytest.raises(ValueError):
+        with coll_ipath.open("w") as handle:
+            handle.read()
+
+    ipath.remove()
+
+
+def test_path_open(session):
+    ipath = IrodsPath(session, "open_test.txt")
+    ipath.remove()
+
+    test_str_1 = "This is a test."
+    test_str_2 = "\nAnother test."
+
+    with ipath.open("w") as handle:
+        handle.write(test_str_1.encode("utf-8"))
+
+    with ipath.open("r") as handle:
+        cur_str = handle.read().decode("utf-8")
+        assert cur_str == test_str_1
+
+    with ipath.open("a") as handle:
+        handle.write(test_str_2.encode("utf-8"))
+
+    with ipath.open("r") as handle:
+        assert handle.read().decode("utf-8") == test_str_1 + test_str_2

docker/irods_client/tests/test_search.py

@@ -0,0 +1,60 @@
+from pytest import mark
+
+from ibridges import IrodsPath, search_data, upload
+from ibridges.search import MetaSearch
+
+
+def _found(search_res, path):
+    for res in search_res:
+        assert isinstance(res, IrodsPath)
+        if str(res) == str(path):
+            return True
+    return False
+
+@mark.parametrize("item_name", ["collection", "dataobject"])
+def test_find_path(session, item_name, request):
+    item = request.getfixturevalue(item_name)
+    ipath = IrodsPath(session, item.path)
+    assert _found(search_data(session, path_pattern=item.name), item.path)
+    assert _found(search_data(session, path_pattern=item.name[:4]+"%"), item.path)
+    assert _found(search_data(session, ipath.parent, item.name), item.path)
+    assert _found(search_data(session, ipath.parent.parent, item.name), item.path)
+    assert _found(search_data(session, IrodsPath(session, "/"), item.name), item.path)
+    pat = item.name[:4] + "%"
+    assert _found(search_data(session, path_pattern=pat), item.path)
+    pat = "%" + item.name[-3:]
+    assert _found(search_data(session, path_pattern=pat), item.path)
+    pat = f"{ipath.parent.name}/%{ipath.name[2:]}"
+    assert _found(search_data(session, IrodsPath(session, "/"), path_pattern=pat), item.path)
+    assert not _found(search_data(session, path_pattern="random_file"), item.path)
+
+
+def test_find_checksum(session, dataobject):
+    ipath = IrodsPath(session, dataobject.path)
+    checksum = ipath.checksum
+    assert _found(search_data(session, IrodsPath(session, "/"), checksum=checksum), ipath)
+    assert not _found(search_data(session, IrodsPath(session, "/"), checksum="sha2:a9s8d7hjas"), ipath)
+    assert _found(search_data(session, IrodsPath(session, "/"), checksum=checksum[:15] + "%"), ipath)
+
+
+@mark.parametrize("metadata,is_found", [
+    (MetaSearch("Author", "Ben"), True),
+    (MetaSearch(units="kg"), True),
+    (MetaSearch(value="10"), True),
+    (MetaSearch(key="Random"), False),
+    (MetaSearch(key="Author", value="10"), False),
+    (MetaSearch(key="Author", units="kg"), False),
+    (MetaSearch(key="Mass", units=None), False),
+    ([MetaSearch(key="Author"), MetaSearch(value="10")], True),
+    ([MetaSearch("Author"), MetaSearch("Random")], False),
+])
+@mark.parametrize("item_name", ["collection", "dataobject"])
+def test_find_meta(session, item_name, request, metadata, is_found):
+    item = request.getfixturevalue(item_name)
+    ipath = IrodsPath(session, item.path)
+    ipath.meta.clear()
+    ipath.meta.add("Author", "Ben")
+    ipath.meta.add("Mass", "10", "kg")
+
+    res = _found(search_data(session, metadata=metadata), ipath)
+    assert res == is_found

docker/irods_client/tests/test_ticket.py

@@ -5,35 +5,35 @@
 from pytest import mark
 
 from ibridges.tickets import Tickets
-
+from ibridges.path import IrodsPath
 
 @mark.parametrize("item_name", ["collection", "dataobject"])
 @mark.parametrize("ticket_type", ["read", "write"])
 @mark.parametrize("n_days_ahead", [1, 100])
 def test_tickets(item_name, ticket_type, n_days_ahead, session, config, request):
     item = request.getfixturevalue(item_name)
-    ipath = item.path
+    ipath = IrodsPath(session, item.path)
     tickets = Tickets(session)
     tickets.clear()
-    assert len(tickets.update_tickets()) == 0
+    assert len(tickets.fetch_tickets()) == 0
 
     exp_date = datetime.datetime.today() + datetime.timedelta(days=n_days_ahead)
-    tickets.create_ticket(str(ipath), ticket_type=ticket_type, expiry_date=exp_date)
-    assert len(tickets.update_tickets()) == 1
+    tickets.create_ticket(ipath, ticket_type=ticket_type, expiry_date=exp_date)
+    assert len(tickets.fetch_tickets()) == 1
     ticket_str = tickets.all_ticket_strings[0]
     tick = tickets.get_ticket(ticket_str)
     assert isinstance(tick, irods.ticket.Ticket)
-    ticket_data = tickets.update_tickets()[0]
+    ticket_data = tickets.fetch_tickets()[0]
     assert ticket_data.name == ticket_str
     assert ticket_data.type == ticket_type
-    assert ticket_data.path == str(ipath)
+    assert str(ticket_data.path) == str(ipath)
 
     # It seems that generally irods invalidates the tickets at midnight of the same day
     if config.get("ticket_date_only", False):
         assert ticket_data.expiration_date.date() == exp_date.date()
     else:
         assert ticket_data.expiration_date == exp_date
     tickets.delete_ticket(tick)
-    assert len(tickets.update_tickets()) == 0
+    assert len(tickets.fetch_tickets()) == 0
     with pytest.raises(KeyError):
         tickets.delete_ticket(tick, check=True)

docker/irods_client/tests/test_util.py

@@ -0,0 +1,29 @@
+from copy import deepcopy
+
+from pytest import mark
+
+from ibridges import IrodsPath, Session, download, upload
+from ibridges.util import calc_checksum, checksums_equal
+
+
+@mark.parametrize(
+    "check_type,checksum", [
+        # There seems some PRC issue with dynamically switching checksums
+        # ("md5", "e313c75f6de6e7cea6c641a99adb18d9"),
+        ("sha2", "sha2:Ys0LhUZdm4jCp83Zy//9Jojs74BzKDrnYYPqqv0MqeU=")]
+)
+def test_calc_checksum(irods_env, config, check_type, checksum, testdata, tmp_path):
+    ienv = deepcopy(irods_env)
+    ienv["irods_default_hash_scheme"] = check_type.upper()
+    with Session(irods_env=ienv, password=config["password"]) as session:
+        ipath_coll = IrodsPath(session, "test_check")
+        ipath_coll.remove()
+        ipath_coll.create_collection(session, ipath_coll)
+        upload(session, testdata / "bunny.rtf", ipath_coll)
+        ipath = ipath_coll / "bunny.rtf"
+        download(session, ipath, tmp_path)
+        assert calc_checksum(ipath, checksum_type=check_type) == checksum
+        assert calc_checksum(tmp_path / "bunny.rtf", checksum_type=check_type) == checksum
+        assert checksums_equal(ipath, tmp_path / "bunny.rtf")
+        ipath_coll.remove()
+

docs/requirements.txt

@@ -1,4 +1,4 @@
-sphinx<8.0.0
+sphinx<9.0.0
 sphinx_inline_tabs
 sphinx-rtd-theme
 sphinxcontrib-napoleon

docs/source/api/user_reference.rst

@@ -55,6 +55,7 @@ Searching for data
     :toctree: generated/
 
     search_data
+    MetaSearch
 
 
 Tickets