Merge pull request #81 from AzureCosmosDB/feature/unicode-json

Fixing unicode output from Utf8JsonWriter

Author: bowencode

Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonSinkTests.cs

@@ -1,4 +1,4 @@
-using Cosmos.DataTransfer.Interfaces;
+using Cosmos.DataTransfer.Interfaces;
 using Microsoft.Extensions.Logging.Abstractions;
 using Newtonsoft.Json;
 
@@ -86,7 +86,6 @@ public async Task WriteAsync_WithSourceDates_PreservesDateFormats()
             Assert.IsTrue(outputData.Any(o => o.Id == 3 && o.Created == randomTime));
         }
 
-
         [TestMethod]
         public async Task WriteAsync_WithDateArray_PreservesDateFormats()
         {
@@ -119,5 +118,45 @@ public async Task WriteAsync_WithDateArray_PreservesDateFormats()
             Assert.AreEqual(DateTime.UnixEpoch, outputData?.Single().Dates?.ElementAt(2));
         }
 
+        [TestMethod]
+        public async Task WriteAsync_WithUnicode_PreservesOriginalCharacters()
+        {
+            var sink = new JsonFileSink();
+
+            const string japanese = "炎ギ因";
+            const string emoji = "✉✔✈";
+            const string text = "XXX";
+            var data = new List<DictionaryDataItem>
+            {
+                new(new Dictionary<string, object?>
+                {
+                    { "Id", 1 },
+                    { "Array", new[] { japanese, emoji, text } },
+                    { "Emoji", emoji },
+                    { japanese, text },
+                })
+            };
+
+            string outputFile = $"{DateTime.UtcNow:yy-MM-dd}_MixedUnicodeOutput.json";
+            var config = TestHelpers.CreateConfig(new Dictionary<string, string>
+            {
+                { "FilePath", outputFile }
+            });
+
+            await sink.WriteAsync(data.ToAsyncEnumerable(), config, new JsonFileSource(), NullLogger.Instance);
+
+            string json = await File.ReadAllTextAsync(outputFile);
+
+            Assert.IsTrue(json.Contains(japanese));
+            Assert.IsTrue(json.Contains(emoji));
+
+            var outputData = JsonConvert.DeserializeObject<List<TestDataObject>>(json);
+
+            var dataObject = outputData?.Single();
+            Assert.AreEqual(japanese, dataObject?.Array?.ElementAt(0));
+            Assert.AreEqual(emoji, dataObject?.Array?.ElementAt(1));
+            Assert.AreEqual(emoji, dataObject?.Emoji);
+            Assert.AreEqual(text, dataObject?.炎ギ因);
+        }
     }
 }

Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/TestDataObject.cs

@@ -1,10 +1,13 @@
-namespace Cosmos.DataTransfer.JsonExtension.UnitTests
+namespace Cosmos.DataTransfer.JsonExtension.UnitTests
 {
     public class TestDataObject
     {
         public int Id { get; set; }
         public string? Name { get; set; }
         public DateTime? Created { get; set; }
         public List<DateTime>? Dates { get; set; }
+        public List<string>? Array { get; set; }
+        public string? Emoji { get; set; }
+        public string? 炎ギ因 { get; set; }
     }
 }

Interfaces/Cosmos.DataTransfer.Interfaces/DataItemJsonConverter.cs

@@ -1,5 +1,6 @@
 using System.Collections;
 using System.Text;
+using System.Text.Encodings.Web;
 using System.Text.Json;
 
 namespace Cosmos.DataTransfer.Interfaces;
@@ -18,11 +19,11 @@ public static string AsJsonString(this IDataItem dataItem, bool indented, bool i
         return Encoding.UTF8.GetString(bytes);
     }
 
-    public static void WriteDataItem(Utf8JsonWriter writer, IDataItem item, bool includeNullFields, string? objectName = null)
+    public static void WriteDataItem(Utf8JsonWriter writer, IDataItem item, bool includeNullFields, JsonEncodedText? objectName = null)
     {
         if (objectName != null)
         {
-            writer.WriteStartObject(objectName);
+            writer.WriteStartObject(objectName.Value);
         }
         else
         {
@@ -40,22 +41,23 @@ public static void WriteDataItem(Utf8JsonWriter writer, IDataItem item, bool inc
 
     private static void WriteFieldValue(Utf8JsonWriter writer, string fieldName, object? fieldValue, bool includeNullFields)
     {
+        var propertyName = GetAsUnescaped(fieldName);
         if (fieldValue == null)
         {
             if (includeNullFields)
             {
-                writer.WriteNull(fieldName);
+                writer.WriteNull(propertyName);
             }
         }
         else
         {
             if (fieldValue is IDataItem child)
             {
-                WriteDataItem(writer, child, includeNullFields, fieldName);
+                WriteDataItem(writer, child, includeNullFields, propertyName);
             }
             else if (fieldValue is not string && fieldValue is IEnumerable children)
             {
-                writer.WriteStartArray(fieldName);
+                writer.WriteStartArray(propertyName);
                 foreach (object arrayItem in children)
                 {
                     if (arrayItem is IDataItem arrayChild)
@@ -76,30 +78,35 @@ private static void WriteFieldValue(Utf8JsonWriter writer, string fieldName, obj
                     }
                     else
                     {
-                        writer.WriteStringValue(arrayItem.ToString());
+                        writer.WriteStringValue(GetAsUnescaped(arrayItem.ToString()!));
                     }
                 }
                 writer.WriteEndArray();
             }
             else if (TryGetNumber(fieldValue, out var number))
             {
-                writer.WriteNumber(fieldName, number);
+                writer.WriteNumber(propertyName, number);
             }
             else if (fieldValue is bool boolean)
             {
-                writer.WriteBoolean(fieldName, boolean);
+                writer.WriteBoolean(propertyName, boolean);
             }
             else if (fieldValue is DateTime date)
             {
-                writer.WriteString(fieldName, date.ToString("O"));
+                writer.WriteString(propertyName, date.ToString("O"));
             }
             else
             {
-                writer.WriteString(fieldName, fieldValue.ToString());
+                writer.WriteString(propertyName, GetAsUnescaped(fieldValue.ToString()!));
             }
         }
     }
 
+    private static JsonEncodedText GetAsUnescaped(string text)
+    {
+        return JsonEncodedText.Encode(text, JavaScriptEncoder.UnsafeRelaxedJsonEscaping);
+    }
+
     private static bool TryGetNumber(object x, out double number)
     {
         if (x is float f)