Skip to content
This repository was archived by the owner on Nov 7, 2023. It is now read-only.

Commit 264fa03

Browse files
bowencodebowencode
authored
Feature/json extension (#178)
* Adding JSON sink and starting tests * Adding extension implementation guide
1 parent 66d9306 commit 264fa03

18 files changed

+383
-2
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ _ReSharper*/
9090
# JustCode is a .NET coding addin-in
9191
.JustCode
9292

93+
# CodeRush is a .NET coding addin-in
94+
.cr
95+
9396
# TeamCity is a build add-in
9497
_TeamCity*
9598

‎Core/Microsoft.DataTransfer.Core/CreatingExtensions.md

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Creating Extensions for the Data Transfer tool
2+
3+
1. Add a new folder in the Extensions folder with the name of your extension.
4+
1. Create the extension project and an accompanying test project.
5+
- The naming convention for extension projects is `Microsoft.DataTransfer.<Name>Extension`.
6+
- Extension projects should use .NET 6 framework and **Console Application** type. A Program.cs file must be included in order to build the console project.
7+
1. Add the new projects to the `CosmosDbDataMigrationTool` solution.
8+
1. In order to facilitate local debugging the extension build output along with any dependencies needs to be copied into the `Core\Microsoft.DataTransfer.Core\bin\Debug\net6.0\Extensions` folder. To set up the project to automatically copy add the following changes.
9+
- Add a Publish Profile to Folder named `LocalDebugFolder` with a Target Location of `..\..\..\Core\Microsoft.DataTransfer.Core\bin\Debug\net6.0\Extensions`
10+
- To publish every time the project builds, edit the .csproj file to add a new post-build step:
11+
12+
```xml
13+
<Target Name="PublishDebug" AfterTargets="Build" Condition=" '$(Configuration)' == 'Debug' ">
14+
<Exec Command="dotnet publish --no-build -p:PublishProfile=LocalDebugFolder" />
15+
</Target>
16+
```
17+
1. Add references to the `System.ComponentModel.Composition` NuGet package and the `Microsoft.DataTransfer.Interfaces` project.
18+
1. Extensions can implement either `IDataSourceExtension` to read data or `IDataSinkExtension` to write data. Classes implementing these interfaces should include a class level `System.ComponentModel.Composition.ExportAttribute` with the implemented interface type as a parameter. This will allow the plugin to get picked up by the main application.
19+
1. Settings needed by the extension can be retrieved from any standard .NET configuration source in the main application by using the `IConfiguration` instance passed into the `ReadAsync` and `WriteAsync` methods. Settings under `SourceSettings`/`SinkSettings` will be included as well as any settings included in JSON files specified by the `SourceSettingsPath`/`SinkSettingsPath`.
20+
1. Implement your extension to read and/or write using the generic `IDataItem` interface which exposes object properties as a list key-value pairs. Depending on the specific structure of the data storage type being implemented, you can choose to support nested objects and arrays or only flat top-level properties.

‎Core/Microsoft.DataTransfer.Core/Samples/JsonSinkSettings.json

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"FilePath": "c:\\temp\\test-json-out.json",
3+
"Indented": true
4+
}
5+

‎Core/Microsoft.DataTransfer.Core/appsettings.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
//"SourceSettingsPath": "C:\\Temp\\JsonSourceSettings.json",
3-
"SinkSettingsPath": "C:\\Temp\\CosmosSinkSettings.json",
3+
"SinkSettingsPath": "C:\\Temp\\JsonSinkSettings.json",
4+
//"SinkSettingsPath": "C:\\Temp\\CosmosSinkSettings.json",
45
"SinkSettings": {
56
"ConnectionString": "<Add in User Secrets>"
67
}

‎CosmosDbDataMigrationTool.sln

+7
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AzureTableAPI", "AzureTable
2525
EndProject
2626
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests", "Extensions\AzureTableAPI\Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests\Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests.csproj", "{F31FADBA-BC78-4839-A9F5-8861695EC95E}"
2727
EndProject
28+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.DataTransfer.JsonExtension.UnitTests", "Extensions\Json\Microsoft.DataTransfer.JsonExtension.UnitTests\Microsoft.DataTransfer.JsonExtension.UnitTests.csproj", "{ED1E375E-A5A3-47EA-A7D5-07344C7E152F}"
29+
EndProject
2830
Global
2931
GlobalSection(SolutionConfigurationPlatforms) = preSolution
3032
Debug|Any CPU = Debug|Any CPU
@@ -63,6 +65,10 @@ Global
6365
{BF19E602-69E2-4F77-95A6-6F76DBB20624}.Debug|Any CPU.Build.0 = Debug|Any CPU
6466
{BF19E602-69E2-4F77-95A6-6F76DBB20624}.Release|Any CPU.ActiveCfg = Release|Any CPU
6567
{BF19E602-69E2-4F77-95A6-6F76DBB20624}.Release|Any CPU.Build.0 = Release|Any CPU
68+
{ED1E375E-A5A3-47EA-A7D5-07344C7E152F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
69+
{ED1E375E-A5A3-47EA-A7D5-07344C7E152F}.Debug|Any CPU.Build.0 = Debug|Any CPU
70+
{ED1E375E-A5A3-47EA-A7D5-07344C7E152F}.Release|Any CPU.ActiveCfg = Release|Any CPU
71+
{ED1E375E-A5A3-47EA-A7D5-07344C7E152F}.Release|Any CPU.Build.0 = Release|Any CPU
6672
EndGlobalSection
6773
GlobalSection(SolutionProperties) = preSolution
6874
HideSolutionNode = FALSE
@@ -75,6 +81,7 @@ Global
7581
{84E51D21-FA45-4CCB-94BB-6B6066749A4A} = {A8A1CEAB-2D82-460C-9B86-74ABD17CD201}
7682
{CB41F880-D0A5-42C1-9571-479CF7E54D82} = {A8A1CEAB-2D82-460C-9B86-74ABD17CD201}
7783
{F31FADBA-BC78-4839-A9F5-8861695EC95E} = {CB41F880-D0A5-42C1-9571-479CF7E54D82}
84+
{ED1E375E-A5A3-47EA-A7D5-07344C7E152F} = {84E51D21-FA45-4CCB-94BB-6B6066749A4A}
7885
EndGlobalSection
7986
GlobalSection(ExtensibilityGlobals) = postSolution
8087
SolutionGuid = {662B3F27-70D8-45E6-A1C0-1438A9C8A542}

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Data/ArraysTypesNesting.json

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
[
2+
{
3+
"id": 1,
4+
"name": "John",
5+
"level2": {
6+
"id": 4324,
7+
"location": "here"
8+
},
9+
"itemsNested": [
10+
{
11+
"number": "one",
12+
"count": 3
13+
},
14+
{
15+
"number": "two",
16+
"count": 7
17+
},
18+
{
19+
"number": "three",
20+
"count": 2
21+
}
22+
]
23+
},
24+
{
25+
"id": 2,
26+
"name": "Matt"
27+
},
28+
{
29+
"noId": 9,
30+
"message": "Needs generated id"
31+
},
32+
{
33+
"id": 3,
34+
"description": "Hello JSON"
35+
}
36+
]

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Data/NestedObjects.json

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
[
2+
{
3+
"id": 1,
4+
"name": "One",
5+
"child": {
6+
"type": "Key",
7+
"data": "Value"
8+
}
9+
},
10+
{
11+
"id": 2,
12+
"name": "Two",
13+
"child": {
14+
"type": "Key",
15+
"data": "Text"
16+
}
17+
},
18+
{
19+
"id": 3,
20+
"name": "Three",
21+
"child": {
22+
"type": "Key",
23+
"data": "String"
24+
}
25+
},
26+
{
27+
"id": 4,
28+
"name": "Four",
29+
"child": {
30+
"type": "Key",
31+
"data": "Number"
32+
}
33+
}
34+
]

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Data/SimpleIdName.json

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[
2+
{
3+
"id": 1,
4+
"name": "One"
5+
},
6+
{
7+
"id": 2,
8+
"name": "Two"
9+
},
10+
{
11+
"id": 3,
12+
"name": "Three"
13+
},
14+
{
15+
"id": 4,
16+
"name": "Four"
17+
}
18+
]

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonRoundTripTests.cs

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
using Newtonsoft.Json.Linq;
2+
3+
namespace Microsoft.DataTransfer.JsonExtension.UnitTests
4+
{
5+
[TestClass]
6+
public class JsonRoundTripTests
7+
{
8+
[TestMethod]
9+
public async Task WriteAsync_fromReadAsync_ProducesIdenticalFile()
10+
{
11+
var input = new JsonDataSourceExtension();
12+
var output = new JsonDataSinkExtension();
13+
14+
const string fileIn = "Data/ArraysTypesNesting.json";
15+
const string fileOut = $"{nameof(WriteAsync_fromReadAsync_ProducesIdenticalFile)}_out.json";
16+
17+
var sourceConfig = TestHelpers.CreateConfig(new Dictionary<string, string>
18+
{
19+
{ "FilePath", fileIn }
20+
});
21+
var sinkConfig = TestHelpers.CreateConfig(new Dictionary<string, string>
22+
{
23+
{ "FilePath", fileOut },
24+
{ "Indented", "true" },
25+
});
26+
27+
await output.WriteAsync(input.ReadAsync(sourceConfig), sinkConfig);
28+
29+
bool areEqual = JToken.DeepEquals(JToken.Parse(await File.ReadAllTextAsync(fileIn)), JToken.Parse(await File.ReadAllTextAsync(fileOut)));
30+
Assert.IsTrue(areEqual);
31+
}
32+
}
33+
}

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSinkTests.cs

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
namespace Microsoft.DataTransfer.JsonExtension.UnitTests
2+
{
3+
[TestClass]
4+
public class JsonSinkTests
5+
{
6+
7+
}
8+
}

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
using Microsoft.Extensions.Configuration;
2+
3+
namespace Microsoft.DataTransfer.JsonExtension.UnitTests
4+
{
5+
[TestClass]
6+
public class JsonSourceTests
7+
{
8+
[TestMethod]
9+
public async Task ReadAsync_WithFlatObjects_ReadsValues()
10+
{
11+
var extension = new JsonDataSourceExtension();
12+
var config = TestHelpers.CreateConfig(new Dictionary<string, string>
13+
{
14+
{ "FilePath", "Data/SimpleIdName.json" }
15+
});
16+
17+
await foreach (var dataItem in extension.ReadAsync(config))
18+
{
19+
CollectionAssert.AreEquivalent(new[] { "id", "name" }, dataItem.GetFieldNames().ToArray());
20+
Assert.IsNotNull(dataItem.GetValue("id"));
21+
Assert.IsNotNull(dataItem.GetValue("name"));
22+
}
23+
}
24+
}
25+
}

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Microsoft.DataTransfer.JsonExtension.UnitTests.csproj

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
8+
<IsPackable>false</IsPackable>
9+
</PropertyGroup>
10+
11+
<ItemGroup>
12+
<PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
13+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
14+
<PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
15+
<PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
16+
<PackageReference Include="coverlet.collector" Version="3.1.2" />
17+
</ItemGroup>
18+
19+
<ItemGroup>
20+
<ProjectReference Include="..\Microsoft.DataTransfer.JsonExtension\Microsoft.DataTransfer.JsonExtension.csproj" />
21+
</ItemGroup>
22+
23+
<ItemGroup>
24+
<None Update="Data\NestedObjects.json">
25+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
26+
</None>
27+
<None Update="Data\SimpleIdName.json">
28+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
29+
</None>
30+
<None Update="Data\ArraysTypesNesting.json">
31+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
32+
</None>
33+
</ItemGroup>
34+
35+
</Project>

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/TestHelpers.cs

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
using Microsoft.Extensions.Configuration;
2+
3+
namespace Microsoft.DataTransfer.JsonExtension.UnitTests
4+
{
5+
public static class TestHelpers
6+
{
7+
public static IConfiguration CreateConfig(Dictionary<string, string> values)
8+
{
9+
return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
10+
}
11+
}
12+
}

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Usings.cs

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
global using Microsoft.VisualStudio.TestTools.UnitTesting;

0 commit comments

Comments
 (0)