Replace uses of urllib.request with requests

src/nidm/experiment/CDE.py

@@ -2,10 +2,10 @@
 from os import environ, path
 import pickle
 import tempfile
-from urllib.request import urlretrieve
 from rdflib import Graph
 from nidm.core import Constants
 import nidm.experiment.Query
+from nidm.util import urlretrieve
 
 
 def download_cde_files():

src/nidm/experiment/Query.py

@@ -37,13 +37,13 @@
 import pickle
 import re
 import tempfile
-from urllib.request import urlretrieve
 import pandas as pd
 import rdflib
 from rdflib import Graph, URIRef, util
 import requests
 from nidm.core import Constants
 import nidm.experiment.CDE
+from nidm.util import urlretrieve
 
 QUERY_CACHE_SIZE = 64
 BIG_CACHE_SIZE = 256

src/nidm/experiment/Utils.py

@@ -4,7 +4,6 @@
 import logging
 import os
 import sys
-from urllib.request import urlopen
 from uuid import UUID
 from cognitiveatlas.api import get_concept, get_disorder
 from datalad.support.annexrepo import AnnexRepo
@@ -966,8 +965,9 @@ def load_nidm_terms_concepts():
     concept_url = "https://raw.githubusercontent.com/NIDM-Terms/terms/master/terms/NIDM_Concepts.jsonld"
 
     try:
-        with urlopen(concept_url) as response:
-            concept_graph = json.loads(response.read().decode("utf-8"))
+        r = requests.get(concept_url)
+        r.raise_for_status()
+        concept_graph = r.json()
     except Exception:
         logging.info("Error opening %s used concepts file..continuing" % concept_url)
         return None

src/nidm/experiment/tools/nidm2bids.py

@@ -44,10 +44,10 @@
 import sys
 import tempfile
 import urllib.parse
-import urllib.request as ur
 import datalad.api as dl
 import pandas as pd
 from rdflib import Graph, URIRef
+import requests
 import validators
 from nidm.core import BIDS_Constants, Constants
 from nidm.core.Constants import DD
@@ -145,10 +145,13 @@ def GetImageFromURL(url):
     # try to open the url and get the pointed to file
     try:
         # open url and get file
-        with ur.urlopen(url) as opener:
+        with requests.get(url, stream=True) as r:
+            r.raise_for_status()
             # write temporary file to disk and use for stats
             with tempfile.NamedTemporaryFile(delete=False) as temp:
-                temp.write(opener.read())
+                for chunk in r.iter_content(65535):
+                    temp.write(chunk)
+                temp.flush()
                 return temp.name
     except Exception:
         print("ERROR! Can't open url: %s" % url)

src/nidm/util.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+from pathlib import Path
+import requests
+
+
+def urlretrieve(url: str, filepath: str | Path) -> None:
+    """`requests`-based alternative to `urllib.request.urlretrieve`"""
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(filepath, "wb") as fp:
+            for chunk in r.iter_content(65535):
+                fp.write(chunk)

tests/experiment/conftest.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 from dataclasses import dataclass
 from typing import Optional
-import urllib.request
 import pytest
 from nidm.experiment.tools.rest import RestParser
+from nidm.util import urlretrieve
 
 # We will test example NIDM files downloaded from
 # the GitHub dbkeator/simple2_NIDM_examples repo
@@ -18,11 +18,11 @@
 @pytest.fixture(scope="session")
 def brain_vol_files(tmp_path_factory: pytest.TempPathFactory) -> list[str]:
     tmp_path = tmp_path_factory.mktemp("brain_vol_files")
-    urllib.request.urlretrieve(
+    urlretrieve(
         "https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/CMU_a/nidm.ttl",
         tmp_path / "cmu_a.nidm.ttl",
     )
-    urllib.request.urlretrieve(
+    urlretrieve(
         "https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/abide/RawDataBIDS/Caltech/nidm.ttl",
         tmp_path / "caltech.nidm.ttl",
     )

tests/experiment/test_navigate.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 from dataclasses import dataclass
 import re
-import urllib.request
 import pytest
 from nidm.experiment import Navigate
+from nidm.util import urlretrieve
 
 
 @dataclass
@@ -22,7 +22,7 @@ def brain_vol(brain_vol_files: list[str]) -> ProjectData:
 @pytest.fixture(scope="module")
 def openneuro(tmp_path_factory: pytest.TempPathFactory) -> ProjectData:
     tmp_path = tmp_path_factory.mktemp("openneuro")
-    urllib.request.urlretrieve(
+    urlretrieve(
         "https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/openneuro/ds000110/nidm.ttl",
         tmp_path / "ds000110.nidm.ttl",
     )

tests/experiment/tools/test_rest.py

@@ -4,7 +4,6 @@
 import re
 import sys
 from typing import Optional
-import urllib.request
 import uuid
 import pytest
 import rdflib
@@ -13,6 +12,7 @@
 from nidm.experiment import Acquisition, AssessmentObject, Project, Query, Session
 from nidm.experiment.CDE import getCDEs
 from nidm.experiment.tools.rest import RestParser
+from nidm.util import urlretrieve
 from ..conftest import BrainVol
 
 if sys.version_info >= (3, 9):
@@ -46,7 +46,7 @@ class OpenNeuro:
 @pytest.fixture(scope="module")
 def openneuro(tmp_path_factory: pytest.TempPathFactory) -> OpenNeuro:
     tmp_path = tmp_path_factory.mktemp("openneuro")
-    urllib.request.urlretrieve(
+    urlretrieve(
         "https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/openneuro/ds000120/nidm.ttl",
         tmp_path / "ds000120.nidm.ttl",
     )

tests/experiment/tools/test_rest_dataelements.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
-import urllib.request
 import pytest
 from nidm.experiment.tools.rest import RestParser
+from nidm.util import urlretrieve
 
 OPENNEURO_FILES = [
     "ds000002.nidm.ttl",
@@ -38,7 +38,7 @@ def openneuro_files(tmp_path_factory: pytest.TempPathFactory) -> list[str]:
     tmp_path = tmp_path_factory.mktemp("openneuro")
     for fname in OPENNEURO_FILES:
         dataset = fname.split(".")[0]
-        urllib.request.urlretrieve(
+        urlretrieve(
             f"https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/openneuro/{dataset}/nidm.ttl",
             tmp_path / fname,
         )

tests/experiment/tools/test_rest_subjects.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 from dataclasses import dataclass
 from typing import Optional
-import urllib.request
 import pytest
 from nidm.experiment.tools.rest import RestParser
+from nidm.util import urlretrieve
 from ..conftest import BrainVol
 
 
@@ -16,7 +16,7 @@ class OpenNeuro:
 @pytest.fixture(scope="module")
 def openneuro(tmp_path_factory: pytest.TempPathFactory) -> OpenNeuro:
     tmp_path = tmp_path_factory.mktemp("openneuro")
-    urllib.request.urlretrieve(
+    urlretrieve(
         "https://raw.githubusercontent.com/dbkeator/simple2_NIDM_examples/master/datasets.datalad.org/openneuro/ds000120/nidm.ttl",
         tmp_path / "ds000120.nidm.ttl",
     )