Skip to content

Commit 325e0a6

Browse files
stefanedwardsstefanedwards
committed
fix: Long values are written as json long values
Addresses issue #152. Relocates DataItemJsonConverter from Interface to Common and adds unit testing with coverage reporting.
1 parent be0966d commit 325e0a6

File tree

15 files changed

+311
-45
lines changed

15 files changed

+311
-45
lines changed

‎Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/Cosmos.DataTransfer.CosmosExtension.UnitTests.csproj

+1-3
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,11 @@
1313
<PackageReference Include="Microsoft.NET.Test.Sdk" />
1414
<PackageReference Include="MSTest.TestAdapter" />
1515
<PackageReference Include="MSTest.TestFramework" />
16+
<PackageReference Include="Newtonsoft.Json" />
1617
<PackageReference Include="coverlet.collector">
1718
<PrivateAssets>all</PrivateAssets>
1819
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
1920
</PackageReference>
20-
<PackageReference Include="Newtonsoft.Json" />
21-
<PackageReference Include="System.Linq.Async" />
22-
<PackageReference Include="System.Text.Json" />
2321
</ItemGroup>
2422

2523
<ItemGroup>

‎Extensions/Csv/Cosmos.DataTransfer.CsvExtension.UnitTests/CsvWriterSettingsTests.cs

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Cosmos.DataTransfer.Interfaces;
66
using Cosmos.DataTransfer.Common.UnitTests;
77
using Moq;
8+
using Cosmos.DataTransfer.Common;
89

910
namespace Cosmos.DataTransfer.CsvExtension.UnitTests;
1011

‎Extensions/Csv/Cosmos.DataTransfer.CsvExtension/CsvFormatReader.cs

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Runtime.CompilerServices;
33
using Cosmos.DataTransfer.CsvExtension.Settings;
44
using Cosmos.DataTransfer.Interfaces;
5+
using Cosmos.DataTransfer.Common;
56
using CsvHelper;
67
using CsvHelper.Configuration;
78
using Microsoft.Extensions.Configuration;

‎Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonFileSinkTests.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using Cosmos.DataTransfer.Interfaces;
1+
using Cosmos.DataTransfer.Common;
22
using Cosmos.DataTransfer.Common.UnitTests;
33
using Microsoft.Extensions.Logging.Abstractions;
44
using Newtonsoft.Json;

‎Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonSinkTests.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using Cosmos.DataTransfer.Interfaces;
1+
using Cosmos.DataTransfer.Common;
22
using Cosmos.DataTransfer.Common.UnitTests;
33
using Microsoft.Extensions.Logging.Abstractions;
44
using Newtonsoft.Json;

‎Extensions/Json/Cosmos.DataTransfer.JsonExtension/JsonFormatWriter.cs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Cosmos.DataTransfer.Interfaces;
2+
using Cosmos.DataTransfer.Common;
23
using Cosmos.DataTransfer.JsonExtension.Settings;
34
using Microsoft.Extensions.Configuration;
45
using Microsoft.Extensions.Logging;

‎Extensions/PostgreSQL/Cosmos.DataTransfer.PostgresqlExtension.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
<ItemGroup>
1818
<ProjectReference Include="..\..\Interfaces\Cosmos.DataTransfer.Interfaces\Cosmos.DataTransfer.Interfaces.csproj" />
19+
<ProjectReference Include="..\..\Interfaces\Cosmos.DataTransfer.Common\Cosmos.DataTransfer.Common.csproj" />
1920
</ItemGroup>
2021
<Target Name="PublishToExtensionsFolder"
2122
AfterTargets="Build"

‎Extensions/PostgreSQL/PostgresqlDataSourceExtension.cs

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Cosmos.DataTransfer.Interfaces;
2+
using Cosmos.DataTransfer.Common;
23
using Cosmos.DataTransfer.PostgresqlExtension.Settings;
34
using Microsoft.Extensions.Configuration;
45
using Microsoft.Extensions.Logging;

‎Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension/Cosmos.DataTransfer.SqlServerExtension.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
<ItemGroup>
2222
<ProjectReference Include="..\..\..\Interfaces\Cosmos.DataTransfer.Interfaces\Cosmos.DataTransfer.Interfaces.csproj" />
23+
<ProjectReference Include="..\..\..\Interfaces\Cosmos.DataTransfer.Common\Cosmos.DataTransfer.Common.csproj" />
2324
</ItemGroup>
2425

2526
<Target Name="PublishToExtensionsFolder"

‎Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension/SqlServerDataSinkExtension.cs

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System.ComponentModel.Composition;
22
using System.Data;
33
using Cosmos.DataTransfer.Interfaces;
4+
using Cosmos.DataTransfer.Common;
45
using Microsoft.Data.SqlClient;
56
using Microsoft.Extensions.Configuration;
67
using Microsoft.Extensions.Logging;

‎Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension/SqlServerDataSourceExtension.cs

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Microsoft.Data.SqlClient;
66
using Microsoft.Extensions.Configuration;
77
using Microsoft.Extensions.Logging;
8+
using Cosmos.DataTransfer.Common;
89

910
namespace Cosmos.DataTransfer.SqlServerExtension
1011
{

‎Interfaces/Cosmos.DataTransfer.Common.UnitTests/DataItemJsonConverterTests.cs

+196
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
using System.Text.Json;
2+
using System.Text.RegularExpressions;
3+
using Microsoft.VisualStudio.TestTools.UnitTesting;
4+
5+
namespace Cosmos.DataTransfer.Common.UnitTests;
6+
7+
[TestClass]
8+
public class DataItemJsonConverterTests
9+
{
10+
[TestMethod]
11+
[DataRow(13, 13, true)]
12+
[DataRow(1L, 1, false)]
13+
[DataRow(2.178f, 1, false)]
14+
[DataRow(2.178, 1, false)]
15+
[DataRow("string", 1, false)]
16+
public void Test_TryGetInteger(object x, int expected, bool success) {
17+
int ret;
18+
var result = DataItemJsonConverter.TryGetInteger(x, out ret);
19+
Assert.AreEqual(success, result);
20+
if (success) {
21+
Assert.AreEqual(expected, ret);
22+
}
23+
}
24+
25+
private static (Utf8JsonWriter,Func<string>) CreateUtf8JsonWriter() {
26+
var stream = new MemoryStream();
27+
var writer = new Utf8JsonWriter(stream, new JsonWriterOptions() {
28+
Indented = false,
29+
SkipValidation = true
30+
});
31+
32+
var read = () => {
33+
writer.Flush();
34+
stream.Seek(0, SeekOrigin.Begin);
35+
using StreamReader reader = new(stream);
36+
var s = reader.ReadToEnd();
37+
stream.Close();
38+
return s;
39+
};
40+
return (writer, read);
41+
}
42+
43+
public static IEnumerable<object?[]> Test_WriteFieldValue_Data { get
44+
{
45+
yield return new object[] { 1, "\"x\":1" };
46+
yield return new object[] { -99, "\"x\":-99" };
47+
yield return new object?[] { null, "" };
48+
yield return new object?[] { null, "\"x\":null", true };
49+
yield return new object[] { 173927362400, "\"x\":173927362400" };
50+
yield return new object[] { -173927362400, "\"x\":-173927362400" };
51+
yield return new object[] { 3ul, "\"x\":3" };
52+
yield return new object[] { 3u, "\"x\":3" };
53+
yield return new object[] { 2.718f, "\"x\":2.7179999351501465" };
54+
yield return new object[] { 2.718, "\"x\":2.718" };
55+
yield return new object[] { 6.022e23, "\"x\":6.022E+23" };
56+
yield return new object[] { 1.66053906892e-27, "\"x\":1.66053906892E-27" };
57+
yield return new object[] { 2.718m, "\"x\":2.718" };
58+
yield return new object[] { -2.718m, "\"x\":-2.718" };
59+
yield return new object[] { (short)2, "\"x\":2" };
60+
yield return new object[] { (short)-2, "\"x\":-2" };
61+
yield return new object[] { (byte)42, "\"x\":42" };
62+
yield return new object[] { (sbyte)-42, "\"x\":-42" };
63+
yield return new object[] { true, "\"x\":true" };
64+
yield return new object[] { false, "\"x\":false" };
65+
yield return new object[] { new DateTime(2025, 10, 15, 16, 45, 12, 666, DateTimeKind.Unspecified), "\"x\":\"2025-10-15T16:45:12.6660000\"" };
66+
yield return new object[] { new DateTime(2025, 10, 15, 16, 45, 12, 666, DateTimeKind.Utc), "\"x\":\"2025-10-15T16:45:12.6660000Z\"" };
67+
yield return new object[] { 'a', "\"x\":\"a\"" };
68+
yield return new object[] { "Greetings human ", "\"x\":\"Greetings human \"" };
69+
// yield return new object[] { true, "\"x\":true" };
70+
} }
71+
72+
[TestMethod]
73+
[DynamicData(nameof(Test_WriteFieldValue_Data))]
74+
public void Test_WriteFieldValue(object? obj, string expected, bool includeNullFields = false) {
75+
var (writer, readFunc) = CreateUtf8JsonWriter();
76+
77+
DataItemJsonConverter.WriteFieldValue(writer, "x", obj, includeNullFields: includeNullFields);
78+
Assert.AreEqual(expected, readFunc(), $"Input type: {obj?.GetType()}.");
79+
}
80+
81+
[TestMethod]
82+
public void Test_WriteFieldValue2() {
83+
var (writer, readFunc) = CreateUtf8JsonWriter();
84+
var obj = (ushort)2;
85+
var expected = "\"x\":2";
86+
87+
DataItemJsonConverter.WriteFieldValue(writer, "x", obj, includeNullFields: false);
88+
Assert.AreEqual(expected, readFunc(), $"Input type: {obj.GetType()}.");
89+
}
90+
91+
[TestMethod]
92+
[DataRow(false)]
93+
[DataRow(true)]
94+
/// null values in arrays are kept regardless of includenullFields.
95+
public void Test_WriteFieldValue_ArrayOfBitsAndPieces(bool includeNullFields) {
96+
var objects = Test_WriteFieldValue_Data
97+
.Where(x => (string)x[1]! != "")
98+
.Select(x => x[0]).ToArray();
99+
Assert.IsNotNull(objects);
100+
var pattern = "\"x\":(.*)$";
101+
var valExtract = Test_WriteFieldValue_Data
102+
.Select(x => Regex.Match((string)x[1]!, pattern).Groups[1].Value)
103+
.Where(x => x.Length > 0)
104+
.ToArray();
105+
var expected = "\"x\":[" + String.Join(',', valExtract) + "]";
106+
107+
108+
var (writer, readFunc) = CreateUtf8JsonWriter();
109+
DataItemJsonConverter.WriteFieldValue(writer, "x", objects, includeNullFields: includeNullFields);
110+
Assert.AreEqual(expected, readFunc(), $"includeNullFields: {includeNullFields}");
111+
}
112+
113+
[TestMethod]
114+
[DataRow(false)]
115+
[DataRow(true)]
116+
public void Test_WriteFieldValue_DataItem(bool includeNullFields) {
117+
var obj = new DictionaryDataItem(new Dictionary<string, object?> {
118+
{ "long", 173927362400 },
119+
{ "NULL", null },
120+
{ "foo", "bar" },
121+
{ "small_pi", 3.1 }
122+
});
123+
var expected = "\"x\":{\"long\":173927362400,\"NULL\":null,\"foo\":\"bar\",\"small_pi\":3.1}";
124+
if (!includeNullFields) {
125+
expected = expected.Replace("\"NULL\":null,", "");
126+
}
127+
var (writer, readFunc) = CreateUtf8JsonWriter();
128+
DataItemJsonConverter.WriteFieldValue(writer, "x", obj, includeNullFields: includeNullFields);
129+
Assert.AreEqual(expected, readFunc(), $"includeNullFields: {includeNullFields}");
130+
}
131+
132+
[TestMethod]
133+
[DataRow(false)]
134+
[DataRow(true)]
135+
public void Test_WriteFieldValue_NestedDataItem(bool includeNullFields) {
136+
var obj = new DictionaryDataItem(new Dictionary<string, object?> {
137+
{ "long", 173927362400 },
138+
{ "NULL", null },
139+
{ "foo", "bar" },
140+
{ "small_pi", 3.1 }
141+
});
142+
obj.Items.Add("obj", new DictionaryDataItem(obj.Items.ToDictionary(x => x.Key, x => x.Value)));
143+
var expected = "\"x\":{\"long\":173927362400,\"NULL\":null,\"foo\":\"bar\",\"small_pi\":3.1," +
144+
"\"obj\":{\"long\":173927362400,\"NULL\":null,\"foo\":\"bar\",\"small_pi\":3.1}}";
145+
if (!includeNullFields) {
146+
expected = expected.Replace("\"NULL\":null,", "");
147+
}
148+
var (writer, readFunc) = CreateUtf8JsonWriter();
149+
DataItemJsonConverter.WriteFieldValue(writer, "x", obj, includeNullFields: includeNullFields);
150+
Assert.AreEqual(expected, readFunc(), $"includeNullFields: {includeNullFields}");
151+
}
152+
153+
[TestMethod]
154+
[DataRow(false)]
155+
[DataRow(true)]
156+
public void Test_WriteFieldValue_ArrayedDataItem(bool includeNullFields) {
157+
var obj = new DictionaryDataItem(new Dictionary<string, object?> {
158+
{ "NULL", null },
159+
{ "small_pi", 3.1 },
160+
{ "arr", new object?[] {
161+
null,
162+
new DictionaryDataItem(new Dictionary<string, object?> {
163+
{ "NULL", null },
164+
{ "foo", "bar" }
165+
})
166+
}}
167+
});
168+
169+
var expected = "\"x\":{\"NULL\":null,\"small_pi\":3.1," +
170+
"\"arr\":[null,{\"NULL\":null,\"foo\":\"bar\"}]}";
171+
if (!includeNullFields) {
172+
expected = expected.Replace("\"NULL\":null,", "");
173+
}
174+
var (writer, readFunc) = CreateUtf8JsonWriter();
175+
DataItemJsonConverter.WriteFieldValue(writer, "x", obj, includeNullFields: includeNullFields);
176+
Assert.AreEqual(expected, readFunc(), $"includeNullFields: {includeNullFields}");
177+
}
178+
179+
[TestMethod]
180+
[DataRow(false)]
181+
[DataRow(true)]
182+
public void Test_AsJsonString(bool includeNullFields) {
183+
var obj = new DictionaryDataItem(new Dictionary<string, object?> {
184+
{ "NULL", null },
185+
{ "small_pi", 3.1 },
186+
});
187+
188+
var expected = "{\"NULL\":null,\"small_pi\":3.1}";
189+
if (!includeNullFields) {
190+
expected = expected.Replace("\"NULL\":null,", "");
191+
}
192+
var json = DataItemJsonConverter.AsJsonString(obj, false, includeNullFields);
193+
Assert.AreEqual(expected, json);
194+
}
195+
}
196+

0 commit comments

Comments
 (0)