Skip to content

Commit def4df0

Browse files
bowencodebowencode
committed
Extending ignoring null to arrays and children; adding tests and docs
1 parent 97db9a5 commit def4df0

File tree

3 files changed

+174
-6
lines changed

3 files changed

+174
-6
lines changed

‎Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosDataSinkExtensionTests.cs

+166
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Cosmos.DataTransfer.Interfaces;
2+
using System.Dynamic;
23

34
namespace Cosmos.DataTransfer.CosmosExtension.UnitTests
45
{
@@ -54,5 +55,170 @@ public void BuildDynamicObjectTree_WithNestedArrays_WorksCorrectly()
5455

5556
Assert.AreEqual("sub2-1", secondSubArray[0].id);
5657
}
58+
59+
[TestMethod]
60+
public void BuildDynamicObjectTree_WithIgnoredNulls_ExcludesNullFields()
61+
{
62+
var item = new CosmosDictionaryDataItem(new Dictionary<string, object?>()
63+
{
64+
{ "id", "1" },
65+
{ "nullField", null },
66+
{
67+
"array",
68+
new List<object?>
69+
{
70+
new List<object?>
71+
{
72+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
73+
{
74+
{ "id", "sub1-1" },
75+
{ "nullField", null },
76+
}),
77+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
78+
{
79+
{ "id", "sub1-2" }
80+
})
81+
},
82+
new List<object?>
83+
{
84+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
85+
{
86+
{ "id", "sub2-1" },
87+
{ "nullField", null },
88+
}),
89+
}
90+
}
91+
},
92+
{ "child1",
93+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
94+
{
95+
{ "id", "child1-1" },
96+
})
97+
},
98+
{ "child2",
99+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
100+
{
101+
{ "id", "child2-1" },
102+
{ "nullField", null },
103+
{ "child2_1",
104+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
105+
{
106+
{ "id", "child2_1-1" },
107+
{ "nullField", null },
108+
})
109+
}
110+
})
111+
}
112+
});
113+
114+
dynamic obj = item.BuildDynamicObjectTree(ignoreNullValues: true)!;
115+
116+
Assert.IsFalse(HasProperty(obj, "nullField"));
117+
118+
Assert.AreEqual(typeof(object[]), obj.array.GetType());
119+
Assert.AreEqual(2, obj.array.Length);
120+
121+
var firstSubArray = obj.array[0];
122+
Assert.AreEqual(typeof(object[]), firstSubArray.GetType());
123+
Assert.IsFalse(HasProperty(firstSubArray[0], "nullField"));
124+
125+
var secondSubArray = obj.array[1];
126+
Assert.AreEqual(typeof(object[]), secondSubArray.GetType());
127+
Assert.IsFalse(HasProperty(secondSubArray[0], "nullField"));
128+
129+
var child2 = obj.child2;
130+
Assert.IsFalse(HasProperty(child2, "nullField"));
131+
Assert.IsFalse(HasProperty(child2.child2_1, "nullField"));
132+
}
133+
134+
[TestMethod]
135+
public void BuildDynamicObjectTree_WithNulls_RetainsNullFields()
136+
{
137+
var item = new CosmosDictionaryDataItem(new Dictionary<string, object?>()
138+
{
139+
{ "id", "1" },
140+
{ "nullField", null },
141+
{
142+
"array",
143+
new List<object?>
144+
{
145+
new List<object?>
146+
{
147+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
148+
{
149+
{ "id", "sub1-1" },
150+
{ "nullField", null },
151+
}),
152+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
153+
{
154+
{ "id", "sub1-2" }
155+
})
156+
},
157+
new List<object?>
158+
{
159+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
160+
{
161+
{ "id", "sub2-1" },
162+
{ "nullField", null },
163+
}),
164+
}
165+
}
166+
},
167+
{ "child1",
168+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
169+
{
170+
{ "id", "child1-1" },
171+
})
172+
},
173+
{ "child2",
174+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
175+
{
176+
{ "id", "child2-1" },
177+
{ "nullField", null },
178+
{ "child2_1",
179+
new CosmosDictionaryDataItem(new Dictionary<string, object?>()
180+
{
181+
{ "id", "child2_1-1" },
182+
{ "nullField", null },
183+
})
184+
}
185+
})
186+
}
187+
});
188+
189+
dynamic obj = item.BuildDynamicObjectTree(ignoreNullValues: false)!;
190+
191+
Assert.IsTrue(HasProperty(obj, "nullField"));
192+
Assert.IsNull(obj.nullField);
193+
194+
Assert.AreEqual(typeof(object[]), obj.array.GetType());
195+
Assert.AreEqual(2, obj.array.Length);
196+
197+
var firstSubArray = obj.array[0];
198+
Assert.AreEqual(typeof(object[]), firstSubArray.GetType());
199+
Assert.IsTrue(HasProperty(firstSubArray[0],"nullField"));
200+
Assert.IsNull(firstSubArray[0].nullField);
201+
Assert.IsFalse(HasProperty(firstSubArray[1], "nullField"));
202+
203+
var secondSubArray = obj.array[1];
204+
Assert.AreEqual(typeof(object[]), secondSubArray.GetType());
205+
Assert.IsTrue(HasProperty(secondSubArray[0], "nullField"));
206+
Assert.IsNull(secondSubArray[0].nullField);
207+
208+
var child2 = obj.child2;
209+
Assert.IsTrue(HasProperty(child2, "nullField"));
210+
Assert.IsNull(child2.nullField);
211+
Assert.IsTrue(HasProperty(child2.child2_1, "nullField"));
212+
Assert.IsNull(child2.child2_1.nullField);
213+
}
214+
215+
public static bool HasProperty(object obj, string name)
216+
{
217+
if (obj is not ExpandoObject)
218+
return obj.GetType().GetProperty(name) != null;
219+
220+
var values = (IDictionary<string, object>)obj;
221+
return values.ContainsKey(name);
222+
}
57223
}
58224
}

‎Extensions/Cosmos/README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ Or with RBAC:
4444
}
4545
```
4646

47-
Sink requires an additional `PartitionKeyPath` parameter which is used when creating the container if it does not exist. To use hierarchical partition keys, instead use the `PartitionKeyPaths` setting to supply an array of up to 3 paths. It also supports an optional `RecreateContainer` parameter (`false` by default) to delete and then recreate the container to ensure only newly imported data is present. The optional `BatchSize` parameter (100 by default) sets the number of items to accumulate before inserting. `ConnectionMode` can be set to either `Gateway` (default) or `Direct` to control how the client connects to the CosmosDB service. For situations where a container is created as part of the transfer operation `CreatedContainerMaxThroughput` (in RUs) and `UseAutoscaleForCreatedContainer` provide the initial throughput settings which will be in effect when executing the transfer. To instead use shared throughput that has been provisioned at the database level, set the `UseSharedThroughput` parameter to `true`. The optional `WriteMode` parameter specifies the type of data write to use: `InsertStream`, `Insert`, `UpsertStream`, or `Upsert`. The `IsServerlessAccount` parameter specifies whether the target account uses Serverless instead of Provisioned throughput, which affects the way containers are created. Additional parameters allow changing the behavior of the Cosmos client appropriate to your environment.
47+
Sink requires an additional `PartitionKeyPath` parameter which is used when creating the container if it does not exist. To use hierarchical partition keys, instead use the `PartitionKeyPaths` setting to supply an array of up to 3 paths. It also supports an optional `RecreateContainer` parameter (`false` by default) to delete and then recreate the container to ensure only newly imported data is present. The optional `BatchSize` parameter (100 by default) sets the number of items to accumulate before inserting. `ConnectionMode` can be set to either `Gateway` (default) or `Direct` to control how the client connects to the CosmosDB service. For situations where a container is created as part of the transfer operation `CreatedContainerMaxThroughput` (in RUs) and `UseAutoscaleForCreatedContainer` provide the initial throughput settings which will be in effect when executing the transfer. To instead use shared throughput that has been provisioned at the database level, set the `UseSharedThroughput` parameter to `true`. The optional `WriteMode` parameter specifies the type of data write to use: `InsertStream`, `Insert`, `UpsertStream`, or `Upsert`. The `IsServerlessAccount` parameter specifies whether the target account uses Serverless instead of Provisioned throughput, which affects the way containers are created. Additional parameters allow changing the behavior of the Cosmos client appropriate to your environment. The `IgnoreNullValues` parameter allows for excluding fields with null values when writing to Cosmos DB.
4848

4949
### Sink
5050

@@ -62,6 +62,7 @@ Sink requires an additional `PartitionKeyPath` parameter which is used when crea
6262
"CreatedContainerMaxThroughput": 1000,
6363
"UseAutoscaleForCreatedContainer": true,
6464
"WriteMode": "InsertStream",
65+
"IgnoreNullValues": false,
6566
"IsServerlessAccount": false,
6667
"UseSharedThroughput": false
6768
}

‎Interfaces/Cosmos.DataTransfer.Interfaces/DataItemExtensions.cs

+6-5
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ public static class DataItemExtensions
99
/// </summary>
1010
/// <param name="source"></param>
1111
/// <param name="requireStringId">If true, adds a new GUID "id" field to any top level items where one is not already present.</param>
12+
/// <param name="ignoreNullValues">If true, excludes fields containing null values from output.</param>
1213
/// <returns>A dynamic object containing the entire data structure.</returns>
1314
/// <remarks>The returned ExpandoObject can be used directly as an IDictionary.</remarks>
1415
public static ExpandoObject? BuildDynamicObjectTree(this IDataItem? source, bool requireStringId = false, bool ignoreNullValues = false)
@@ -50,28 +51,28 @@ public static class DataItemExtensions
5051
}
5152
else if (value is IDataItem child)
5253
{
53-
value = BuildDynamicObjectTree(child);
54+
value = BuildDynamicObjectTree(child, ignoreNullValues: ignoreNullValues);
5455
}
5556
else if (value is IEnumerable<object?> array)
5657
{
57-
value = BuildArray(array);
58+
value = BuildArray(array, ignoreNulls: ignoreNullValues);
5859
}
5960

6061
item.TryAdd(fieldName, value);
6162
}
6263

6364
return item;
6465

65-
static object BuildArray(IEnumerable<object?> array)
66+
static object BuildArray(IEnumerable<object?> array, bool ignoreNulls)
6667
{
6768
return array.Select(dataItem =>
6869
{
6970
switch (dataItem)
7071
{
7172
case IDataItem childObject:
72-
return BuildDynamicObjectTree(childObject);
73+
return BuildDynamicObjectTree(childObject, ignoreNullValues: ignoreNulls);
7374
case IEnumerable<object?> array:
74-
return BuildArray(array);
75+
return BuildArray(array, ignoreNulls);
7576
default:
7677
return dataItem;
7778
}

0 commit comments

Comments
 (0)