Merge pull request #65 from RemyBoyer/cosmos-sink-nested-array

Fix nested arrays in Cosmos SQL Sink

Author: bowencode

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosDataSinkExtensionTests.cs

@@ -0,0 +1,56 @@
+namespace Cosmos.DataTransfer.CosmosExtension.UnitTests
+{
+    [TestClass]
+    public class CosmosDataSinkExtensionTests
+    {
+        [TestMethod]
+        public void BuildObject_WithNestedArrays_WorksCorrectly()
+        {
+            var item = new CosmosDictionaryDataItem(new Dictionary<string, object?>()
+            {
+                {
+                    "array",
+                    new List<object?>
+                    {
+                        new List<object?>
+                        {
+                            new CosmosDictionaryDataItem(new Dictionary<string, object?>()
+                            {
+                                { "id", "sub1-1" }
+                            }),
+                            new CosmosDictionaryDataItem(new Dictionary<string, object?>()
+                            {
+                                { "id", "sub1-2" }
+                            })
+                        },
+                        new List<object?>
+                        {
+                            new CosmosDictionaryDataItem(new Dictionary<string, object?>()
+                            {
+                                { "id", "sub2-1" }
+                            }),
+                        }
+                    }
+                }
+            });
+
+            dynamic obj = CosmosDataSinkExtension.BuildObject(item)!;
+
+            Assert.AreEqual(typeof(object[]), obj.array.GetType());
+            Assert.AreEqual(2, obj.array.Length);
+
+            var firstSubArray = obj.array[0];
+            Assert.AreEqual(typeof(object[]), firstSubArray.GetType());
+            Assert.AreEqual(2, firstSubArray.Length);
+
+            Assert.AreEqual("sub1-1", firstSubArray[0].id);
+            Assert.AreEqual("sub1-2", firstSubArray[1].id);
+
+            var secondSubArray = obj.array[1];
+            Assert.AreEqual(typeof(object[]), secondSubArray.GetType());
+            Assert.AreEqual(1, secondSubArray.Length);
+
+            Assert.AreEqual("sub2-1", secondSubArray[0].id);
+        }
+    }
+}

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/Cosmos.DataTransfer.CosmosExtension.csproj

@@ -7,6 +7,10 @@
 		<OutputType>Exe</OutputType>
 	</PropertyGroup>
 
+	<ItemGroup>
+		<InternalsVisibleTo Include="Cosmos.DataTransfer.CosmosExtension.UnitTests" />
+	</ItemGroup>
+
 	<ItemGroup>
 		<PackageReference Include="Azure.Identity" Version="1.6.0" />
 		<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.34.0" />

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs

@@ -176,7 +176,7 @@ private static MemoryStream CreateItemStream(ExpandoObject item)
             return ((IDictionary<string, object?>)item)[propertyName]?.ToString();
         }
 
-        private static ExpandoObject? BuildObject(IDataItem? source, bool requireStringId = false)
+        internal static ExpandoObject? BuildObject(IDataItem? source, bool requireStringId = false)
         {
             if (source == null)
                 return null;
@@ -202,20 +202,29 @@ private static MemoryStream CreateItemStream(ExpandoObject item)
                 }
                 else if (value is IEnumerable<object?> array)
                 {
-                    value = array.Select(dataItem =>
-                    {
-                        if (dataItem is IDataItem childObject)
-                        {
-                            return BuildObject(childObject);
-                        }
-                        return dataItem;
-                    }).ToArray();
+                    value = BuildArray(array);
                 }
 
                 item.TryAdd(fieldName, value);
             }
 
             return item;
+
+            static object BuildArray(IEnumerable<object?> array)
+            {
+                return array.Select(dataItem =>
+                {
+                    if (dataItem is IDataItem childObject)
+                    {
+                        return BuildObject(childObject);
+                    }
+                    else if (dataItem is IEnumerable<object?> array)
+                    {
+                        return BuildArray(array);
+                    }
+                    return dataItem;
+                }).ToArray();
+            }
         }
 
         public IEnumerable<IDataExtensionSettings> GetSettings()