Merge pull request #186 from Aiven-Open/giacomo-alzetta-aiven-allow-p…

…ydantic-upgrade

Make rohmu compatible with Pydantic V2 using the .v1 namespace

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "httplib2",
     "oauth2client",
     "paramiko",
-    "pydantic < 2",
+    "pydantic >= 1.10.17",
     "python-dateutil",
     "python-snappy",
     "python-swiftclient",

rohmu/common/models.py

@@ -4,7 +4,7 @@
 from typing import Optional
 
 import enum
-import pydantic
+import pydantic.v1 as pyd
 
 
 class StorageOperation(StrEnum):
@@ -42,7 +42,7 @@ class StorageDriver(StrEnum):
     swift = "swift"
 
 
-class RohmuModel(pydantic.BaseModel):
+class RohmuModel(pyd.BaseModel):
     class Config:
         # As we're keen to both export and decode json, just using
         # enum values for encode/decode is much saner than the default
@@ -67,14 +67,14 @@ class ProxyInfo(RohmuModel):
     port: int
     type: ProxyType
     user: Optional[str]
-    password: Optional[str] = pydantic.Field(None, alias="pass")
+    password: Optional[str] = pyd.Field(None, alias="pass")
 
     class Config(RohmuModel.Config):
         # Allow ProxyInfo(**proxy_info.dict()) to work with the alias
         allow_population_by_field_name = True
 
 
-class StorageModel(pydantic.BaseModel):
+class StorageModel(pyd.BaseModel):
     storage_type: StorageDriver
     statsd_info: Optional[StatsdConfig] = None
 

rohmu/common/statsd.py

@@ -18,7 +18,7 @@
 from rohmu.common.strenum import StrEnum
 from typing import AsyncIterator, Dict, Iterator, Optional, Union
 
-import pydantic
+import pydantic.v1 as pyd
 import socket
 import time
 
@@ -31,7 +31,7 @@ class MessageFormat(StrEnum):
 Tags = Dict[str, Union[int, str, None]]
 
 
-class StatsdConfig(pydantic.BaseModel):
+class StatsdConfig(pyd.BaseModel):
     host: str = "127.0.0.1"
     port: int = 8125
     message_format: MessageFormat = MessageFormat.telegraf

rohmu/delta/common.py

@@ -7,7 +7,7 @@
 from enum import Enum
 from multiprocessing.dummy import Pool
 from pathlib import Path
-from pydantic import BaseModel, Field
+from pydantic.v1 import BaseModel, Field
 from rohmu.dates import now
 from rohmu.typing import AnyPath, HasRead
 from types import TracebackType

rohmu/object_storage/config.py

@@ -7,7 +7,7 @@
 
 from enum import Enum, unique
 from pathlib import Path
-from pydantic import Field, root_validator, validator
+from pydantic.v1 import Field, root_validator, validator
 from rohmu.common.models import ProxyInfo, StorageDriver, StorageModel
 from typing import Any, Dict, Final, Literal, Optional, TypeVar
 

test/object_storage/test_azure.py

@@ -94,24 +94,30 @@ def test_get_contents_to_fileobj_raises_error_on_invalid_byte_range(mock_get_blo
 
 
 def test_minimal_config() -> None:
-    config = AzureObjectStorageConfig(account_name="test")
+    config = AzureObjectStorageConfig(account_name="test", bucket_name=None, account_key=None, sas_token=None)
     assert config.account_name == "test"
 
 
 def test_azure_config_host_port_set_together() -> None:
     with pytest.raises(ValueError):
-        AzureObjectStorageConfig(account_name="test", host="localhost")
+        AzureObjectStorageConfig(account_name="test", host="localhost", bucket_name=None, account_key=None, sas_token=None)
     with pytest.raises(ValueError):
-        AzureObjectStorageConfig(account_name="test", port=10000)
-    config = AzureObjectStorageConfig(account_name="test", host="localhost", port=10000)
+        AzureObjectStorageConfig(account_name="test", port=10000, bucket_name=None, account_key=None, sas_token=None)
+    config = AzureObjectStorageConfig(
+        account_name="test", host="localhost", port=10000, bucket_name=None, account_key=None, sas_token=None
+    )
     assert config.host == "localhost"
     assert config.port == 10000
 
 
 def test_valid_azure_cloud_endpoint() -> None:
     with pytest.raises(ValueError):
-        AzureObjectStorageConfig(account_name="test", azure_cloud="invalid")
-    config = AzureObjectStorageConfig(account_name="test", azure_cloud="public")
+        AzureObjectStorageConfig(
+            account_name="test", azure_cloud="invalid", bucket_name=None, account_key=None, sas_token=None
+        )
+    config = AzureObjectStorageConfig(
+        account_name="test", azure_cloud="public", bucket_name=None, account_key=None, sas_token=None
+    )
     assert config.azure_cloud == "public"
 
 

test/object_storage/test_s3.py

@@ -6,7 +6,7 @@
 from datetime import datetime
 from io import BytesIO
 from pathlib import Path
-from pydantic import ValidationError
+from pydantic.v1 import ValidationError
 from rohmu.common.models import StorageOperation
 from rohmu.errors import InvalidByteRangeError
 from rohmu.object_storage.base import TransferWithConcurrentUploadSupport
@@ -282,6 +282,8 @@ def test_validate_is_verify_tls_and_cert_path() -> None:
             region="test-region",
             bucket_name="test-bucket",
             cert_path=Path("test_cert_path"),
+            aws_secret_access_key=None,
+            aws_session_token=None,
         )
     assert "cert_path is set but is_verify_tls is False" in str(e.value)
 

test/test_factory.py

@@ -1,3 +1,4 @@
+from rohmu import ProxyInfo
 from rohmu.factory import Config, get_class_for_transfer, get_transfer, get_transfer_from_model
 from rohmu.object_storage.config import S3ObjectStorageConfig
 from rohmu.object_storage.s3 import S3Transfer
@@ -80,13 +81,17 @@ def test_get_transfer_from_model(
     config = S3ObjectStorageConfig(
         region="dummy-region",
         bucket_name="dummy-bucket",
-        proxy_info={
-            "host": "proxy.test",
-            "port": "16666",
-            "type": "socks5",
-            "user": "bob",
-            "pass": "secret",
-        },
+        proxy_info=ProxyInfo.parse_obj(
+            {
+                "host": "proxy.test",
+                "port": "16666",
+                "type": "socks5",
+                "user": "bob",
+                "pass": "secret",
+            }
+        ),
+        aws_secret_access_key=None,
+        aws_session_token=None,
     )
     get_transfer_from_model(config)
     create_s3_client.assert_called_once_with(
@@ -106,13 +111,17 @@ def test_get_transfer_serialized_model(
     config = S3ObjectStorageConfig(
         region="dummy-region",
         bucket_name="dummy-bucket",
-        proxy_info={
-            "host": "proxy.test",
-            "port": "16666",
-            "type": "socks5",
-            "user": "bob",
-            "pass": "secret",
-        },
+        proxy_info=ProxyInfo.parse_obj(
+            {
+                "host": "proxy.test",
+                "port": "16666",
+                "type": "socks5",
+                "user": "bob",
+                "pass": "secret",
+            }
+        ),
+        aws_secret_access_key=None,
+        aws_session_token=None,
     )
     get_transfer(config.dict())
     create_s3_client.assert_called_once_with(