Skip to content

Commit b891e81

Browse files
bowencodebowencode
committed
SQL Source and Sink extension (#191)
* Adding basic outline of Sql Extension * Adding initial working version of SQL source * Adding sink implementation for SQL * Adding direct reference to common package version * Adding docs and JSON column support
1 parent 3a6131a commit b891e81

File tree

22 files changed

+522
-120
lines changed

22 files changed

+522
-120
lines changed

‎Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj

+3
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,12 @@
99
</PropertyGroup>
1010

1111
<ItemGroup>
12+
<PackageReference Include="Azure.Core" Version="1.25.0" />
13+
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.0.0" />
1214
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="6.0.1" />
1315
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
1416
<PackageReference Include="System.ComponentModel.Composition" Version="6.0.0" />
17+
<PackageReference Include="System.Configuration.ConfigurationManager" Version="6.0.0" />
1518
</ItemGroup>
1619

1720
<ItemGroup>

‎Core/Microsoft.DataTransfer.Core/Program.cs

+5-5
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public static async Task Main(string[] args)
2828
{
2929
Directory.CreateDirectory("Extensions");
3030
}
31-
catalog.Catalogs.Add(new DirectoryCatalog("Extensions"));
31+
catalog.Catalogs.Add(new DirectoryCatalog("Extensions", "*Extension.dll"));
3232
var container = new CompositionContainer(catalog);
3333

3434
var sources = LoadExtensions<IDataSourceExtension>(container);
@@ -37,10 +37,10 @@ public static async Task Main(string[] args)
3737
Console.WriteLine($"{sources.Count + sinks.Count} Extensions Loaded");
3838

3939
var source = GetExtensionSelection(options.Source, sources, "Source");
40-
var sourceConfig = BuildSettingsConfiguration(configuration, options.SourceSettingsPath, $"{source.DisplayName}SourceSettings");
40+
var sourceConfig = BuildSettingsConfiguration(configuration, options.SourceSettingsPath, $"{source.DisplayName}SourceSettings", options.Source == null);
4141

4242
var sink = GetExtensionSelection(options.Sink, sinks, "Sink");
43-
var sinkConfig = BuildSettingsConfiguration(configuration, options.SinkSettingsPath, $"{sink.DisplayName}SinkSettings");
43+
var sinkConfig = BuildSettingsConfiguration(configuration, options.SinkSettingsPath, $"{sink.DisplayName}SinkSettings", options.Sink == null);
4444

4545
var data = source.ReadAsync(sourceConfig);
4646
await sink.WriteAsync(data, sinkConfig);
@@ -97,14 +97,14 @@ private static T GetExtensionSelection<T>(string? selectionName, List<T> extensi
9797
return extensions[input - 1];
9898
}
9999

100-
private static IConfiguration BuildSettingsConfiguration(IConfiguration configuration, string? settingsPath, string configSection)
100+
private static IConfiguration BuildSettingsConfiguration(IConfiguration configuration, string? settingsPath, string configSection, bool promptForFile)
101101
{
102102
IConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
103103
if (!string.IsNullOrEmpty(settingsPath))
104104
{
105105
configurationBuilder = configurationBuilder.AddJsonFile(settingsPath);
106106
}
107-
else
107+
else if (promptForFile)
108108
{
109109
Console.Write($"Load settings from a file? (y/n):");
110110
var response = Console.ReadLine();

‎Core/Microsoft.DataTransfer.Core/Properties/launchSettings.json

+8
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,14 @@
1111
"Cosmos->JSON": {
1212
"commandName": "Project",
1313
"commandLineArgs": "--source cosmos --sink json --SourceSettingsPath=c:\\temp\\CosmosSourceSettings.json --SinkSettingsPath=c:\\temp\\JsonSinkSettings.json"
14+
},
15+
"SqlServer->Cosmos": {
16+
"commandName": "Project",
17+
"commandLineArgs": "--source sqlServer --sink cosmos --SourceSettingsPath=c:\\temp\\SqlSourceSettings.json --SinkSettingsPath=c:\\temp\\CosmosSinkSettings.json"
18+
},
19+
"JSON->SqlServer": {
20+
"commandName": "Project",
21+
"commandLineArgs": "--source json --sink sqlServer --JsonSourceSettings:FilePath=c:\\temp\\test-json-sql-in.json --SinkSettingsPath=c:\\temp\\SqlSinkSettings.json"
1422
}
1523
}
1624
}

‎Core/Microsoft.DataTransfer.Core/appsettings.json

+6
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,11 @@
1616
"AzureTableAPISourceSettings": {
1717
},
1818
"AzureTableAPISinkSettings": {
19+
},
20+
"SqlServerSourceSettings": {
21+
"ConnectionString": "<Add in User Secrets>"
22+
},
23+
"SqlServerSinkSettings": {
24+
"ConnectionString": "<Add in User Secrets>"
1925
}
2026
}

‎Core/Microsoft.DataTransfer.Core/runtimeconfig.template.json

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"additionalProbingPaths": [
3+
"Extensions"
4+
]
5+
}

‎CosmosDbDataMigrationTool.sln

+17
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DataTransfer.Json
3232
EndProject
3333
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DataTransfer.CosmosExtension.UnitTests", "Extensions\Cosmos\Microsoft.DataTransfer.CosmosExtension.UnitTests\Microsoft.DataTransfer.CosmosExtension.UnitTests.csproj", "{C7A3910D-A7F6-4767-9A7A-19CFA4CCB7A8}"
3434
EndProject
35+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SqlServer", "SqlServer", "{2F075279-E8B0-4A6E-A8D9-2058B7DEC671}"
36+
EndProject
37+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DataTransfer.SqlServerExtension", "Extensions\SqlServer\Microsoft.DataTransfer.SqlServerExtension\Microsoft.DataTransfer.SqlServerExtension.csproj", "{7A020621-77E6-4DD1-B230-50A46B4BB2B1}"
38+
EndProject
39+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DataTransfer.SqlServerExtension.UnitTests", "Extensions\SqlServer\Microsoft.DataTransfer.SqlServerExtension.UnitTests\Microsoft.DataTransfer.SqlServerExtension.UnitTests.csproj", "{3E4C4ABF-D8C2-4997-A719-E756483C8D63}"
40+
EndProject
3541
Global
3642
GlobalSection(SolutionConfigurationPlatforms) = preSolution
3743
Debug|Any CPU = Debug|Any CPU
@@ -78,6 +84,14 @@ Global
7884
{C7A3910D-A7F6-4767-9A7A-19CFA4CCB7A8}.Debug|Any CPU.Build.0 = Debug|Any CPU
7985
{C7A3910D-A7F6-4767-9A7A-19CFA4CCB7A8}.Release|Any CPU.ActiveCfg = Release|Any CPU
8086
{C7A3910D-A7F6-4767-9A7A-19CFA4CCB7A8}.Release|Any CPU.Build.0 = Release|Any CPU
87+
{7A020621-77E6-4DD1-B230-50A46B4BB2B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
88+
{7A020621-77E6-4DD1-B230-50A46B4BB2B1}.Debug|Any CPU.Build.0 = Debug|Any CPU
89+
{7A020621-77E6-4DD1-B230-50A46B4BB2B1}.Release|Any CPU.ActiveCfg = Release|Any CPU
90+
{7A020621-77E6-4DD1-B230-50A46B4BB2B1}.Release|Any CPU.Build.0 = Release|Any CPU
91+
{3E4C4ABF-D8C2-4997-A719-E756483C8D63}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
92+
{3E4C4ABF-D8C2-4997-A719-E756483C8D63}.Debug|Any CPU.Build.0 = Debug|Any CPU
93+
{3E4C4ABF-D8C2-4997-A719-E756483C8D63}.Release|Any CPU.ActiveCfg = Release|Any CPU
94+
{3E4C4ABF-D8C2-4997-A719-E756483C8D63}.Release|Any CPU.Build.0 = Release|Any CPU
8195
EndGlobalSection
8296
GlobalSection(SolutionProperties) = preSolution
8397
HideSolutionNode = FALSE
@@ -94,6 +108,9 @@ Global
94108
{F6EAC33B-9F7D-433B-9328-622FB8938C24} = {F18E789A-D32D-48D3-B75F-1196D7215F74}
95109
{ED1E375E-A5A3-47EA-A7D5-07344C7E152F} = {84E51D21-FA45-4CCB-94BB-6B6066749A4A}
96110
{C7A3910D-A7F6-4767-9A7A-19CFA4CCB7A8} = {D0475879-F47F-4D12-9A43-812043CFFD22}
111+
{2F075279-E8B0-4A6E-A8D9-2058B7DEC671} = {A8A1CEAB-2D82-460C-9B86-74ABD17CD201}
112+
{7A020621-77E6-4DD1-B230-50A46B4BB2B1} = {2F075279-E8B0-4A6E-A8D9-2058B7DEC671}
113+
{3E4C4ABF-D8C2-4997-A719-E756483C8D63} = {2F075279-E8B0-4A6E-A8D9-2058B7DEC671}
97114
EndGlobalSection
98115
GlobalSection(ExtensibilityGlobals) = postSolution
99116
SolutionGuid = {662B3F27-70D8-45E6-A1C0-1438A9C8A542}

‎Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs

+5-3
Original file line numberDiff line numberDiff line change
@@ -105,18 +105,20 @@ private static Task<int> InsertItemAsync(Container container, ExpandoObject item
105105
if (source == null)
106106
return null;
107107

108-
var fields = source.GetFieldNames();
108+
var fields = source.GetFieldNames().ToList();
109109
var item = new ExpandoObject();
110-
if (requireStringId && !fields.Contains("id"))
110+
if (requireStringId && !fields.Contains("id", StringComparer.CurrentCultureIgnoreCase))
111111
{
112112
item.TryAdd("id", Guid.NewGuid().ToString());
113113
}
114114
foreach (string field in fields)
115115
{
116116
object? value = source.GetValue(field);
117+
var fieldName = field;
117118
if (string.Equals(field, "id", StringComparison.CurrentCultureIgnoreCase) && requireStringId)
118119
{
119120
value = value?.ToString();
121+
fieldName = "id";
120122
}
121123
else if (value is IDataItem child)
122124
{
@@ -134,7 +136,7 @@ private static Task<int> InsertItemAsync(Container container, ExpandoObject item
134136
}).ToArray();
135137
}
136138

137-
item.TryAdd(field, value);
139+
item.TryAdd(fieldName, value);
138140
}
139141

140142
return item;

‎Extensions/Json/Microsoft.DataTransfer.JsonExtension/JsonDataSinkExtension.cs

+1-112
Original file line numberDiff line numberDiff line change
@@ -29,123 +29,12 @@ public async Task WriteAsync(IAsyncEnumerable<IDataItem> dataItems, IConfigurati
2929

3030
await foreach (var item in dataItems.WithCancellation(cancellationToken))
3131
{
32-
WriteDataItem(settings, writer, item);
32+
DataItemJsonConverter.WriteDataItem(writer, item, settings.IncludeNullFields);
3333
}
3434

3535
writer.WriteEndArray();
3636
Console.WriteLine($"Completed writing data to file '{settings.FilePath}'");
3737
}
3838
}
39-
40-
private static void WriteDataItem(JsonSinkSettings settings, Utf8JsonWriter writer, IDataItem item, string? objectName = null)
41-
{
42-
if (objectName != null)
43-
{
44-
writer.WriteStartObject(objectName);
45-
}
46-
else
47-
{
48-
writer.WriteStartObject();
49-
}
50-
51-
foreach (string fieldName in item.GetFieldNames())
52-
{
53-
var fieldValue = item.GetValue(fieldName);
54-
WriteFieldValue(writer, fieldName, fieldValue, settings);
55-
}
56-
57-
writer.WriteEndObject();
58-
}
59-
60-
private static void WriteFieldValue(Utf8JsonWriter writer, string fieldName, object? fieldValue, JsonSinkSettings settings)
61-
{
62-
if (fieldValue == null)
63-
{
64-
if (settings.IncludeNullFields)
65-
{
66-
writer.WriteNull(fieldName);
67-
}
68-
}
69-
else
70-
{
71-
if (fieldValue is IDataItem child)
72-
{
73-
WriteDataItem(settings, writer, child, fieldName);
74-
}
75-
else if (fieldValue is IEnumerable<object> children)
76-
{
77-
writer.WriteStartArray(fieldName);
78-
foreach (object arrayItem in children)
79-
{
80-
if (arrayItem is IDataItem arrayChild)
81-
{
82-
WriteDataItem(settings, writer, arrayChild);
83-
}
84-
else if(TryGetNumber(arrayItem, out var number))
85-
{
86-
writer.WriteNumberValue(number);
87-
}
88-
else if (arrayItem is bool boolean)
89-
{
90-
writer.WriteBooleanValue(boolean);
91-
}
92-
else
93-
{
94-
writer.WriteStringValue(arrayItem.ToString());
95-
}
96-
}
97-
writer.WriteEndArray();
98-
}
99-
else if (TryGetNumber(fieldValue, out var number))
100-
{
101-
writer.WriteNumber(fieldName, number);
102-
}
103-
else if (fieldValue is bool boolean)
104-
{
105-
writer.WriteBoolean(fieldName, boolean);
106-
}
107-
else
108-
{
109-
writer.WriteString(fieldName, fieldValue.ToString());
110-
}
111-
}
112-
}
113-
114-
private static bool TryGetNumber(object x, out double number)
115-
{
116-
if (x is float f)
117-
{
118-
number = f;
119-
return true;
120-
}
121-
if (x is double d)
122-
{
123-
number = d;
124-
return true;
125-
}
126-
if (x is decimal m)
127-
{
128-
number = (double) m;
129-
return true;
130-
}
131-
if (x is int i)
132-
{
133-
number = i;
134-
return true;
135-
}
136-
if (x is short s)
137-
{
138-
number = s;
139-
return true;
140-
}
141-
if (x is long l)
142-
{
143-
number = l;
144-
return true;
145-
}
146-
147-
number = default;
148-
return false;
149-
}
15039
}
15140
}

‎Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension.UnitTests/Data/FlatFile.json

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[
2+
{
3+
"id": 1,
4+
"name": "One"
5+
},
6+
{
7+
"id": 2,
8+
"name": "Two"
9+
},
10+
{
11+
"id": 3,
12+
"name": "Three"
13+
},
14+
{
15+
"id": 4,
16+
"name": "Four"
17+
}
18+
]

‎Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension.UnitTests/Microsoft.DataTransfer.SqlServerExtension.UnitTests.csproj

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
8+
<IsPackable>false</IsPackable>
9+
</PropertyGroup>
10+
11+
<ItemGroup>
12+
<PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
13+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
14+
<PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
15+
<PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
16+
<PackageReference Include="coverlet.collector" Version="3.1.2" />
17+
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
18+
</ItemGroup>
19+
20+
<ItemGroup>
21+
<ProjectReference Include="..\Microsoft.DataTransfer.SqlServerExtension\Microsoft.DataTransfer.SqlServerExtension.csproj" />
22+
</ItemGroup>
23+
24+
<ItemGroup>
25+
<None Update="Data\FlatFile.json">
26+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
27+
</None>
28+
</ItemGroup>
29+
30+
</Project>

‎Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension.UnitTests/SqlServerExtensionTests.cs

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
namespace Microsoft.DataTransfer.SqlServerExtension.UnitTests
3+
{
4+
[TestClass]
5+
public class SqlServerExtensionTests
6+
{
7+
}
8+
}

‎Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension.UnitTests/Usings.cs

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
global using Microsoft.VisualStudio.TestTools.UnitTesting;

‎Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/ColumnMapping.cs

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
using System.ComponentModel.DataAnnotations;
2+
3+
namespace Microsoft.DataTransfer.SqlServerExtension
4+
{
5+
public class ColumnMapping
6+
{
7+
[Required]
8+
public string? ColumnName { get; set; }
9+
public string? SourceFieldName { get; set; }
10+
public bool AllowNull { get; set; } = true;
11+
public object? DefaultValue { get; set; }
12+
13+
public string? GetFieldName()
14+
{
15+
return SourceFieldName ?? ColumnName;
16+
}
17+
}
18+
}

‎Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Microsoft.DataTransfer.SqlServerExtension.csproj

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
<OutputType>Exe</OutputType>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.0.0" />
12+
<PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
13+
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="6.0.0" />
14+
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
15+
<PackageReference Include="System.ComponentModel.Composition" Version="6.0.0" />
16+
<PackageReference Include="System.Interactive.Async" Version="6.0.1" />
17+
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
18+
</ItemGroup>
19+
20+
<ItemGroup>
21+
<ProjectReference Include="..\..\..\Interfaces\Microsoft.DataTransfer.Interfaces\Microsoft.DataTransfer.Interfaces.csproj" />
22+
</ItemGroup>
23+
24+
<Target Name="PublishDebug" AfterTargets="Build" Condition=" '$(Configuration)' == 'Debug' ">
25+
<Exec Command="dotnet publish --no-build -p:PublishProfile=LocalDebugFolder" />
26+
</Target>
27+
</Project>

‎Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Program.cs

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Console.WriteLine();

0 commit comments

Comments
 (0)