Validation cleanup

bowencode · bowencode · commit d76dc504274c · 2023-05-27T14:58:15.000-07:00
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs
@@ -5,6 +5,11 @@ namespace Cosmos.DataTransfer.CosmosExtension.UnitTests;
 [TestClass]
 public class CosmosSinkSettingsTests
 {
+    private static void LogErrors(IEnumerable<string?> errors)
+    {
+        foreach (var error in errors) Console.WriteLine($"Validation Error: {error}");
+    }
+
     [TestMethod]
     public void GetValidationErrors_WithNoConnection_ReturnsError()
     {
@@ -15,6 +20,7 @@ public void GetValidationErrors_WithNoConnection_ReturnsError()
         };
 
         var validationErrors = settings.GetValidationErrors();
+        LogErrors(validationErrors);
 
         Assert.AreEqual(1, validationErrors.Count(v => v.Contains(nameof(CosmosSinkSettings.ConnectionString))));
     }
@@ -30,6 +36,7 @@ public void GetValidationErrors_WithNoRbacConnection_ReturnsError()
         };
 
         var validationErrors = settings.GetValidationErrors();
+        LogErrors(validationErrors);
 
         Assert.AreEqual(1, validationErrors.Count(v => v.Contains(nameof(CosmosSinkSettings.AccountEndpoint))));
     }
@@ -55,7 +62,7 @@ public void Validate_WithAccountEndpoint_Succeeds()
             UseRbacAuth = true,
             AccountEndpoint = "https://localhost:8081/",
             Database = "db",
-            Container = "container",
+            Container = "container"
         };
 
         settings.Validate();
@@ -70,11 +77,31 @@ public void GetValidationErrors_WhenRecreateContainerTrue_RequiresPartitionKeyPa
             Database = "db",
             Container = "container",
             RecreateContainer = true,
+            WriteMode = DataWriteMode.Insert,
         };
 
         var validationErrors = settings.GetValidationErrors();
+        LogErrors(validationErrors);
 
-        Assert.AreEqual(1, validationErrors.Count(v => v.Contains(nameof(CosmosSinkSettings.PartitionKeyPath))));
+        Assert.AreEqual(1, validationErrors.Count(v => v.Contains(nameof(CosmosSinkSettings.PartitionKeyPath)) && v.Contains(nameof(CosmosSinkSettings.RecreateContainer))));
+    }
+
+    [TestMethod]
+    public void GetValidationErrors_WhenWriteModeIsStream_RequiresPartitionKeyPath()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            RecreateContainer = false,
+            WriteMode = DataWriteMode.InsertStream,
+        };
+
+        var validationErrors = settings.GetValidationErrors();
+        LogErrors(validationErrors);
+
+        Assert.AreEqual(1, validationErrors.Count(v => v.Contains(nameof(CosmosSinkSettings.PartitionKeyPath)) && v.Contains(nameof(CosmosSinkSettings.WriteMode))));
     }
 
     [TestMethod]
@@ -86,8 +113,46 @@ public void GetValidationErrors_WhenDbContainerMissing_ReturnsErrors()
         };
 
         var validationErrors = settings.GetValidationErrors();
+        LogErrors(validationErrors);
 
         Assert.AreEqual(1, validationErrors.Count(v => v.Contains(nameof(CosmosSinkSettings.Database))));
         Assert.AreEqual(1, validationErrors.Count(v => v.Contains(nameof(CosmosSinkSettings.Container))));
     }
+
+    [TestMethod]
+    public void GetValidationErrors_WhenRecreateContainerTrueAndWriteModeStreamWithPartitionKeys_Succeeds()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            RecreateContainer = true,
+            WriteMode = DataWriteMode.InsertStream,
+            PartitionKeyPaths = new List<string> { "/a", "/b" },
+        };
+
+        var validationErrors = settings.GetValidationErrors();
+        LogErrors(validationErrors);
+
+        Assert.AreEqual(0, validationErrors.Count());
+    }
+
+    [TestMethod]
+    public void GetValidationErrors_WhenPartitionKeysInvalid_ReturnsErrors()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            RecreateContainer = true,
+            PartitionKeyPaths = new List<string> { "a", "b" },
+        };
+
+        var validationErrors = settings.GetValidationErrors();
+        LogErrors(validationErrors);
+
+        Assert.AreEqual(1, validationErrors.Count(v => v.Contains(nameof(CosmosSinkSettings.PartitionKeyPaths))));
+    }
 }
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs
@@ -46,19 +46,21 @@ public async Task WriteAsync(IAsyncEnumerable<IDataItem> dataItems, IConfigurati
                     catch { }
                 }
 
-            var containerProperties = new ContainerProperties
-            {
-                Id = settings.Container,
-                PartitionKeyDefinitionVersion = PartitionKeyDefinitionVersion.V2,
-            };
-            if (settings.PartitionKeyPaths != null)
-            {
-                containerProperties.PartitionKeyPaths = settings.PartitionKeyPaths;
-            }
-            else
-            {
-                containerProperties.PartitionKeyPath = settings.PartitionKeyPath;
-            }
+                var containerProperties = new ContainerProperties
+                {
+                    Id = settings.Container,
+                    PartitionKeyDefinitionVersion = PartitionKeyDefinitionVersion.V2,
+                };
+
+                if (settings.PartitionKeyPaths != null)
+                {
+                    logger.LogInformation("Using partition key paths: {PartitionKeyPaths}", string.Join(", ", settings.PartitionKeyPaths));
+                    containerProperties.PartitionKeyPaths = settings.PartitionKeyPaths;
+                }
+                else
+                {
+                    containerProperties.PartitionKeyPath = settings.PartitionKeyPath;
+                }
 
                 ThroughputProperties? throughputProperties = settings.IsServerlessAccount
                     ? null
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSinkSettings.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSinkSettings.cs
@@ -1,8 +1,5 @@
-﻿using System.Collections.Generic;
-using System.ComponentModel.DataAnnotations;
+﻿using System.ComponentModel.DataAnnotations;
 using Cosmos.DataTransfer.Interfaces;
-using Cosmos.DataTransfer.Interfaces.Manifest;
-using Microsoft.Azure.Cosmos;
 
 namespace Cosmos.DataTransfer.CosmosExtension
 {
@@ -16,7 +13,7 @@ public class CosmosSinkSettings : CosmosSettingsBase, IDataExtensionSettings
         public int? CreatedContainerMaxThroughput { get; set; }
         public bool UseAutoscaleForCreatedContainer { get; set; } = true;
         public bool IsServerlessAccount { get; set; } = false;
-        public DataWriteMode WriteMode { get; set; } = DataWriteMode.InsertStream;
+        public DataWriteMode WriteMode { get; set; } = DataWriteMode.Insert;
         public List<string>? PartitionKeyPaths { get; set; }
 
         public override IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
@@ -33,20 +30,42 @@ public override IEnumerable<ValidationResult> Validate(ValidationContext validat
                     yield return new ValidationResult("RBAC auth does not support Container creation", new[] { nameof(UseRbacAuth) });
                 }
 
-                if (string.IsNullOrWhiteSpace(PartitionKeyPath))
+                if (MissingPartitionKeys())
                 {
-                    yield return new ValidationResult("PartitionKeyPath must be specified when RecreateContainer is true", new[] { nameof(PartitionKeyPath) });
+                    yield return new ValidationResult("PartitionKeyPath must be specified when RecreateContainer is true", new[] { nameof(PartitionKeyPath), nameof(PartitionKeyPaths) });
                 }
             }
-            if (!string.IsNullOrWhiteSpace(PartitionKeyPath) && !PartitionKeyPath.StartsWith("/"))
+
+            if (PartitionKeyPaths?.Any(p => !string.IsNullOrEmpty(p)) == true)
+            {
+                if (PartitionKeyPaths.Any(p => !p.StartsWith("/")))
+                {
+                    yield return new ValidationResult("PartitionKeyPaths values must start with /", new[] { nameof(PartitionKeyPaths) });
+                }
+            }
+            else if (!string.IsNullOrWhiteSpace(PartitionKeyPath))
             {
-                yield return new ValidationResult("PartitionKeyPath must start with /", new[] { nameof(PartitionKeyPath) });
+                if (!PartitionKeyPath.StartsWith("/"))
+                {
+                    yield return new ValidationResult("PartitionKeyPath must start with /", new[] { nameof(PartitionKeyPath) });
+                }
             }
 
-            if (string.IsNullOrWhiteSpace(PartitionKeyPath) && WriteMode is DataWriteMode.InsertStream or DataWriteMode.UpsertStream)
+            if (MissingPartitionKeys() && WriteMode is DataWriteMode.InsertStream or DataWriteMode.UpsertStream)
             {
-                yield return new ValidationResult("PartitionKeyPath must be specified when WriteMode is set to InsertStream or UpsertStream", new[] { nameof(PartitionKeyPath), nameof(WriteMode) });
+                yield return new ValidationResult("PartitionKeyPath must be specified when WriteMode is set to InsertStream or UpsertStream", new[] { nameof(PartitionKeyPath), nameof(PartitionKeyPaths), nameof(WriteMode) });
             }
         }
+
+        private bool MissingPartitionKeys()
+        {
+            if (!string.IsNullOrWhiteSpace(PartitionKeyPath))
+                return false;
+
+            if (PartitionKeyPaths?.Any(p => !string.IsNullOrEmpty(p)) == true)
+                return false;
+
+            return true;
+        }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,5 @@`
`1`		`-using System.Collections.Generic;`
`2`		`-using System.ComponentModel.DataAnnotations;`
	`1`	`+using System.ComponentModel.DataAnnotations;`
`3`	`2`	`using Cosmos.DataTransfer.Interfaces;`
`4`		`-using Cosmos.DataTransfer.Interfaces.Manifest;`
`5`		`-using Microsoft.Azure.Cosmos;`
`6`	`3`
`7`	`4`	`namespace Cosmos.DataTransfer.CosmosExtension`
`8`	`5`	`{`
`@@ -16,7 +13,7 @@ public class CosmosSinkSettings : CosmosSettingsBase, IDataExtensionSettings`
`16`	`13`	`public int? CreatedContainerMaxThroughput { get; set; }`
`17`	`14`	`public bool UseAutoscaleForCreatedContainer { get; set; } = true;`
`18`	`15`	`public bool IsServerlessAccount { get; set; } = false;`
`19`		`- public DataWriteMode WriteMode { get; set; } = DataWriteMode.InsertStream;`
	`16`	`+ public DataWriteMode WriteMode { get; set; } = DataWriteMode.Insert;`
`20`	`17`	`public List<string>? PartitionKeyPaths { get; set; }`
`21`	`18`
`22`	`19`	`public override IEnumerable<ValidationResult> Validate(ValidationContext validationContext)`
`@@ -33,20 +30,42 @@ public override IEnumerable<ValidationResult> Validate(ValidationContext validat`
`33`	`30`	`yield return new ValidationResult("RBAC auth does not support Container creation", new[] { nameof(UseRbacAuth) });`
`34`	`31`	`}`
`35`	`32`
`36`		`- if (string.IsNullOrWhiteSpace(PartitionKeyPath))`
	`33`	`+ if (MissingPartitionKeys())`
`37`	`34`	`{`
`38`		`- yield return new ValidationResult("PartitionKeyPath must be specified when RecreateContainer is true", new[] { nameof(PartitionKeyPath) });`
	`35`	`+ yield return new ValidationResult("PartitionKeyPath must be specified when RecreateContainer is true", new[] { nameof(PartitionKeyPath), nameof(PartitionKeyPaths) });`
`39`	`36`	`}`
`40`	`37`	`}`
`41`		`- if (!string.IsNullOrWhiteSpace(PartitionKeyPath) && !PartitionKeyPath.StartsWith("/"))`
	`38`	`+`
	`39`	`+ if (PartitionKeyPaths?.Any(p => !string.IsNullOrEmpty(p)) == true)`
	`40`	`+ {`
	`41`	`+ if (PartitionKeyPaths.Any(p => !p.StartsWith("/")))`
	`42`	`+ {`
	`43`	`+ yield return new ValidationResult("PartitionKeyPaths values must start with /", new[] { nameof(PartitionKeyPaths) });`
	`44`	`+ }`
	`45`	`+ }`
	`46`	`+ else if (!string.IsNullOrWhiteSpace(PartitionKeyPath))`
`42`	`47`	`{`
`43`		`- yield return new ValidationResult("PartitionKeyPath must start with /", new[] { nameof(PartitionKeyPath) });`
	`48`	`+ if (!PartitionKeyPath.StartsWith("/"))`
	`49`	`+ {`
	`50`	`+ yield return new ValidationResult("PartitionKeyPath must start with /", new[] { nameof(PartitionKeyPath) });`
	`51`	`+ }`
`44`	`52`	`}`
`45`	`53`
`46`		`- if (string.IsNullOrWhiteSpace(PartitionKeyPath) && WriteMode is DataWriteMode.InsertStream or DataWriteMode.UpsertStream)`
	`54`	`+ if (MissingPartitionKeys() && WriteMode is DataWriteMode.InsertStream or DataWriteMode.UpsertStream)`
`47`	`55`	`{`
`48`		`- yield return new ValidationResult("PartitionKeyPath must be specified when WriteMode is set to InsertStream or UpsertStream", new[] { nameof(PartitionKeyPath), nameof(WriteMode) });`
	`56`	`+ yield return new ValidationResult("PartitionKeyPath must be specified when WriteMode is set to InsertStream or UpsertStream", new[] { nameof(PartitionKeyPath), nameof(PartitionKeyPaths), nameof(WriteMode) });`
`49`	`57`	`}`
`50`	`58`	`}`
	`59`	`+`
	`60`	`+ private bool MissingPartitionKeys()`
	`61`	`+ {`
	`62`	`+ if (!string.IsNullOrWhiteSpace(PartitionKeyPath))`
	`63`	`+ return false;`
	`64`	`+`
	`65`	`+ if (PartitionKeyPaths?.Any(p => !string.IsNullOrEmpty(p)) == true)`
	`66`	`+ return false;`
	`67`	`+`
	`68`	`+ return true;`
	`69`	`+ }`
`51`	`70`	`}`
`52`	`71`	`}`