Skip to content

Commit ac09156

Browse files
bowencodebowencode
committed
Merge update
2 parents aaa1ed9 + 4347fb8 commit ac09156

File tree

67 files changed

+1642
-77
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+1642
-77
lines changed

‎.github/actions/build-with-plugins/action.yml

+15
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,21 @@ runs:
103103
-p:PublishReadyToRun=false \
104104
-p:PublishTrimmed=false \
105105
-p:Version=0.0.${{ github.run_number }}
106+
- name: Build Parquet Extension
107+
shell: bash
108+
run: |
109+
dotnet publish \
110+
Extensions/Parquet/Cosmos.DataTransfer.ParquetExtension/Cosmos.DataTransfer.ParquetExtension.csproj \
111+
--configuration Release \
112+
--output ${{ inputs.platform-short }}/Extensions \
113+
--self-contained false \
114+
--runtime ${{ inputs.runtime }} \
115+
-p:PublishSingleFile=false \
116+
-p:DebugType=embedded \
117+
-p:EnableCompressionInSingleFile=true \
118+
-p:PublishReadyToRun=false \
119+
-p:PublishTrimmed=false \
120+
-p:Version=0.0.${{ github.run_number }}
106121
- name: Upload package
107122
uses: actions/upload-artifact@v3
108123
with:

‎.github/workflows/dotnet-build-test.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: .NET
1+
name: Build and Test all .NET projects
22

33
on:
44
push:
@@ -8,9 +8,9 @@ on:
88

99
jobs:
1010
build:
11-
11+
name: Build and test .NET projects
1212
runs-on: ubuntu-latest
13-
13+
container: mcr.microsoft.com/dotnet/sdk:6.0
1414
steps:
1515
- uses: actions/checkout@v3
1616
- name: Setup .NET
@@ -28,4 +28,4 @@ jobs:
2828
uses: actions/upload-artifact@v3
2929
with:
3030
name: debug-build
31-
path: /home/runner/work/azure-documentdb-datamigrationtool/azure-documentdb-datamigrationtool/Core/Cosmos.DataTransfer.Core/bin/Debug/net6.0 #path/to/artifact/ # or path/to/artifact
31+
path: /home/runner/work/data-migration-desktop-tool/data-migration-desktop-tool/Core/Cosmos.DataTransfer.Core/bin/Debug/net6.0 #path/to/artifact/ # or path/to/artifact

‎.github/workflows/validate.yml

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
name: Validate all .NET projects
22
on:
3-
pull_request:
4-
branches:
5-
- main
3+
# pull_request:
4+
# branches:
5+
# - main
6+
workflow_dispatch:
67
jobs:
78
build-test:
89
name: Build and test .NET projects

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -433,3 +433,4 @@ FodyWeavers.xsd
433433

434434
# macOS .DS_Store
435435
**/.DS_Store
436+
/Core/Cosmos.DataTransfer.Core/migrationsettings.json

‎Core/Cosmos.DataTransfer.Core/Cosmos.DataTransfer.Core.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
</PropertyGroup>
2020

2121
<ItemGroup>
22-
<PackageReference Include="Azure.Core" Version="1.25.0" />
22+
<PackageReference Include="Azure.Core" Version="1.31.0" />
2323
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.0.0" />
2424
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="6.0.1" />
2525
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />

‎Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json

+10-2
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,19 @@
1818
},
1919
"JSON->SqlServer": {
2020
"commandName": "Project",
21-
"commandLineArgs": "run --source json --sink sqlServer --SourceSettings:FilePath=c:\\temp\\test-json-sql-in.json --SettingsPath=c:\\temp\\Json-SqlSettings.json"
21+
"commandLineArgs": "run --source json --sink sqlServer --SourceSettings:FilePath=c:\\temp\\test-json-sql-in.json --settings=c:\\temp\\Json-SqlSettings.json"
2222
},
2323
"JSON URI->Cosmos": {
2424
"commandName": "Project",
25-
"commandLineArgs": "--source json --sink cosmos-nosql --SourceSettings:FilePath=https://raw.githubusercontent.com/AzureCosmosDB/data-migration-desktop-tool/feature/cosmos-configuration/Extensions/Json/.JsonExtension.UnitTests/Data/ArraysTypesNesting.json --SettingsPath=c:\\temp\\CosmosSinkSettings.json"
25+
"commandLineArgs": "--source json-file --sink cosmos-nosql --SourceSettings:FilePath=https://raw.githubusercontent.com/AzureCosmosDB/data-migration-desktop-tool/main/Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/Data/ArraysTypesNesting.json --settings=c:\\temp\\CosmosSinkSettings.json"
26+
},
27+
"JSON->Parquet": {
28+
"commandName": "Project",
29+
"commandLineArgs": "run -from json-file(beta) --sink parquet --settings C:\\Temp\\Json-Parquet.json"
30+
},
31+
"Parquet->JSON": {
32+
"commandName": "Project",
33+
"commandLineArgs": "run --sink json-file(beta) --source parquet --settings C:\\Temp\\Parquet-Json.json"
2634
}
2735
}
2836
}

‎Core/Cosmos.DataTransfer.Core/migrationsettings.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{
22
"Source": null,
33
"Sink": null,
4-
"SourceSettings": {
4+
"SourceSettings": {
55
},
6-
"SinkSettings": {
6+
"SinkSettings": {
77
},
88
"Operations": [
99
//{

‎CosmosDbDataMigrationTool.sln

+36
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,20 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.SqlServ
5454
EndProject
5555
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.SqlServerExtension.UnitTests", "Extensions\SqlServer\Cosmos.DataTransfer.SqlServerExtension.UnitTests\Cosmos.DataTransfer.SqlServerExtension.UnitTests.csproj", "{3E4C4ABF-D8C2-4997-A719-E756483C8D63}"
5656
EndProject
57+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Parquet", "Parquet", "{2BD63BEE-CEE8-4FF3-8A2E-373D9F4E2FE8}"
58+
EndProject
59+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.ParquetExtension", "Extensions\Parquet\Cosmos.DataTransfer.ParquetExtension\Cosmos.DataTransfer.ParquetExtension.csproj", "{B83FEBC7-08B6-4C25-831A-EA0C08794F67}"
60+
EndProject
61+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.Common", "Interfaces\Cosmos.DataTransfer.Common\Cosmos.DataTransfer.Common.csproj", "{0FAD9D89-2E41-4D65-8440-5C01885D9292}"
62+
EndProject
63+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AzureBlob", "AzureBlob", "{9627A42A-BEB0-4A39-B49C-C3C6D54E705A}"
64+
EndProject
65+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AwsS3", "AwsS3", "{502197E4-F554-4B5B-9235-FBFE7E49EBEF}"
66+
EndProject
67+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Cosmos.DataTransfer.AwsS3Storage", "Extensions\AwsS3\Cosmos.DataTransfer.AwsS3Storage\Cosmos.DataTransfer.AwsS3Storage.csproj", "{8BA59E9C-0B45-426F-A672-61D40C3C4FB7}"
68+
EndProject
69+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Cosmos.DataTransfer.AzureBlobStorage", "Extensions\AzureBlob\Cosmos.DataTransfer.AzureBlobStorage\Cosmos.DataTransfer.AzureBlobStorage.csproj", "{60ACD837-40BD-4596-832A-139CCBFA7EFE}"
70+
EndProject
5771
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CognitiveSearch", "CognitiveSearch", "{F745B535-C483-4894-8BA9-657DB1913D0B}"
5872
ProjectSection(SolutionItems) = preProject
5973
Extensions\CognitiveSearch\README.md = Extensions\CognitiveSearch\README.md
@@ -117,6 +131,22 @@ Global
117131
{3E4C4ABF-D8C2-4997-A719-E756483C8D63}.Debug|Any CPU.Build.0 = Debug|Any CPU
118132
{3E4C4ABF-D8C2-4997-A719-E756483C8D63}.Release|Any CPU.ActiveCfg = Release|Any CPU
119133
{3E4C4ABF-D8C2-4997-A719-E756483C8D63}.Release|Any CPU.Build.0 = Release|Any CPU
134+
{B83FEBC7-08B6-4C25-831A-EA0C08794F67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
135+
{B83FEBC7-08B6-4C25-831A-EA0C08794F67}.Debug|Any CPU.Build.0 = Debug|Any CPU
136+
{B83FEBC7-08B6-4C25-831A-EA0C08794F67}.Release|Any CPU.ActiveCfg = Release|Any CPU
137+
{B83FEBC7-08B6-4C25-831A-EA0C08794F67}.Release|Any CPU.Build.0 = Release|Any CPU
138+
{0FAD9D89-2E41-4D65-8440-5C01885D9292}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
139+
{0FAD9D89-2E41-4D65-8440-5C01885D9292}.Debug|Any CPU.Build.0 = Debug|Any CPU
140+
{0FAD9D89-2E41-4D65-8440-5C01885D9292}.Release|Any CPU.ActiveCfg = Release|Any CPU
141+
{0FAD9D89-2E41-4D65-8440-5C01885D9292}.Release|Any CPU.Build.0 = Release|Any CPU
142+
{8BA59E9C-0B45-426F-A672-61D40C3C4FB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
143+
{8BA59E9C-0B45-426F-A672-61D40C3C4FB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
144+
{8BA59E9C-0B45-426F-A672-61D40C3C4FB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
145+
{8BA59E9C-0B45-426F-A672-61D40C3C4FB7}.Release|Any CPU.Build.0 = Release|Any CPU
146+
{60ACD837-40BD-4596-832A-139CCBFA7EFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
147+
{60ACD837-40BD-4596-832A-139CCBFA7EFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
148+
{60ACD837-40BD-4596-832A-139CCBFA7EFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
149+
{60ACD837-40BD-4596-832A-139CCBFA7EFE}.Release|Any CPU.Build.0 = Release|Any CPU
120150
{37DBC3CB-F8F6-48F8-BFBA-07A27D2E9DD9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
121151
{37DBC3CB-F8F6-48F8-BFBA-07A27D2E9DD9}.Debug|Any CPU.Build.0 = Debug|Any CPU
122152
{37DBC3CB-F8F6-48F8-BFBA-07A27D2E9DD9}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -144,6 +174,12 @@ Global
144174
{2F075279-E8B0-4A6E-A8D9-2058B7DEC671} = {A8A1CEAB-2D82-460C-9B86-74ABD17CD201}
145175
{7A020621-77E6-4DD1-B230-50A46B4BB2B1} = {2F075279-E8B0-4A6E-A8D9-2058B7DEC671}
146176
{3E4C4ABF-D8C2-4997-A719-E756483C8D63} = {2F075279-E8B0-4A6E-A8D9-2058B7DEC671}
177+
{2BD63BEE-CEE8-4FF3-8A2E-373D9F4E2FE8} = {A8A1CEAB-2D82-460C-9B86-74ABD17CD201}
178+
{B83FEBC7-08B6-4C25-831A-EA0C08794F67} = {2BD63BEE-CEE8-4FF3-8A2E-373D9F4E2FE8}
179+
{9627A42A-BEB0-4A39-B49C-C3C6D54E705A} = {A8A1CEAB-2D82-460C-9B86-74ABD17CD201}
180+
{502197E4-F554-4B5B-9235-FBFE7E49EBEF} = {A8A1CEAB-2D82-460C-9B86-74ABD17CD201}
181+
{8BA59E9C-0B45-426F-A672-61D40C3C4FB7} = {502197E4-F554-4B5B-9235-FBFE7E49EBEF}
182+
{60ACD837-40BD-4596-832A-139CCBFA7EFE} = {9627A42A-BEB0-4A39-B49C-C3C6D54E705A}
147183
{F745B535-C483-4894-8BA9-657DB1913D0B} = {A8A1CEAB-2D82-460C-9B86-74ABD17CD201}
148184
{37DBC3CB-F8F6-48F8-BFBA-07A27D2E9DD9} = {F745B535-C483-4894-8BA9-657DB1913D0B}
149185
{B55A183D-E4D4-4095-8569-D361A6AA1D10} = {F745B535-C483-4894-8BA9-657DB1913D0B}

‎ExampleConfigs.md

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Example `migrationsettings.json` Files
2+
3+
## JSON to Cosmos-NoSQL
4+
```json
5+
{
6+
"Source": "json",
7+
"Sink": "cosmos-nosql",
8+
"SourceSettings": {
9+
"FilePath": "https://mytestfiles.local/sales-data.json"
10+
},
11+
"SinkSettings": {
12+
"ConnectionString": "AccountEndpoint=https://...",
13+
"Database": "myDb",
14+
"Container": "myContainer",
15+
"PartitionKeyPath": "/id",
16+
"RecreateContainer": true,
17+
"WriteMode": "Insert",
18+
"CreatedContainerMaxThroughput": 5000,
19+
"IsServerlessAccount": false
20+
}
21+
}
22+
```
23+
24+
## Cosmos-NoSQL to JSON
25+
```json
26+
{
27+
"Source": "Cosmos-NoSql",
28+
"Sink": "JSON",
29+
"SourceSettings":
30+
{
31+
"ConnectionString": "AccountEndpoint=https://...",
32+
"Database":"cosmicworks",
33+
"Container":"customers",
34+
"IncludeMetadataFields": true
35+
},
36+
"SinkSettings":
37+
{
38+
"FilePath": "c:\\data\\cosmicworks\\customers.json",
39+
"Indented": true
40+
}
41+
}
42+
```
43+
44+
## MongoDB to Cosmos-NoSQL
45+
```json
46+
{
47+
"Source": "mongodb",
48+
"Sink": "cosmos-nosql",
49+
"SourceSettings": {
50+
"ConnectionString": "mongodb://...",
51+
"DatabaseName": "sales",
52+
"Collection": "person"
53+
},
54+
"SinkSettings": {
55+
"ConnectionString": "AccountEndpoint=https://...",
56+
"Database": "users",
57+
"Container": "migrated",
58+
"PartitionKeyPath": "/id",
59+
"ConnectionMode": "Direct",
60+
"WriteMode": "UpsertStream",
61+
"CreatedContainerMaxThroughput": 8000,
62+
"UseAutoscaleForCreatedContainer": false
63+
}
64+
}
65+
```
66+
67+
## SqlServer to AzureTableAPI
68+
```json
69+
{
70+
"Source": "SqlServer",
71+
"Sink": "AzureTableApi",
72+
"SourceSettings": {
73+
"ConnectionString": "Server=...",
74+
"QueryText": "SELECT Id, Date, Amount FROM dbo.Payments WHERE Status = 'open'"
75+
},
76+
"SinkSettings": {
77+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=...",
78+
"Table": "payments",
79+
"RowKeyFieldName": "Id"
80+
}
81+
}
82+
```
83+
84+
## Cosmos-NoSQL to SqlServer
85+
```json
86+
{
87+
"Source": "cosmos-nosql",
88+
"Sink": "sqlserver",
89+
"SourceSettings":
90+
{
91+
"ConnectionString": "AccountEndpoint=https://...",
92+
"Database":"operations",
93+
"Container":"alerts",
94+
"PartitionKeyValue": "jan",
95+
"Query": "SELECT a.name, a.description, a.count, a.id, a.isSet FROM a"
96+
},
97+
"SinkSettings":
98+
{
99+
"ConnectionString": "Server=...",
100+
"TableName": "Import",
101+
"ColumnMappings": [
102+
{
103+
"ColumnName": "Name"
104+
},
105+
{
106+
"ColumnName": "Description"
107+
},
108+
{
109+
"ColumnName": "Count",
110+
"SourceFieldName": "number"
111+
},
112+
{
113+
"ColumnName": "Id"
114+
},
115+
{
116+
"ColumnName": "IsSet",
117+
"AllowNull": false,
118+
"DefaultValue": false
119+
}
120+
]
121+
}
122+
}
123+
```

‎Extensions/AwsS3/Cosmos.DataTransfer.AwsS3Storage/AwsS3DataSink.cs

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using Cosmos.DataTransfer.Interfaces;
2+
using Microsoft.Extensions.Configuration;
3+
using Microsoft.Extensions.Logging;
4+
5+
namespace Cosmos.DataTransfer.AwsS3Storage
6+
{
7+
public class AwsS3DataSink : IComposableDataSink
8+
{
9+
public async Task WriteToTargetAsync(Func<Stream, Task> writeToStream, IConfiguration config, IDataSourceExtension dataSource, ILogger logger, CancellationToken cancellationToken = default)
10+
{
11+
var settings = config.Get<AwsS3SinkSettings>();
12+
settings.Validate();
13+
14+
logger.LogInformation("Saving file to AWS S3 Bucket '{BucketName}'", settings.S3BucketName);
15+
16+
S3Writer.InitializeS3Client(settings.S3AccessKey, settings.S3SecretKey, settings.S3Region);
17+
await using var stream = new MemoryStream();
18+
await writeToStream(stream);
19+
await S3Writer.WriteToS3(settings.S3BucketName, stream, settings.FileName, cancellationToken);
20+
}
21+
}
22+
}

‎Extensions/AwsS3/Cosmos.DataTransfer.AwsS3Storage/AwsS3SinkSettings.cs

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
using Cosmos.DataTransfer.Interfaces;
2+
using System.ComponentModel.DataAnnotations;
3+
4+
namespace Cosmos.DataTransfer.AwsS3Storage
5+
{
6+
public class AwsS3SinkSettings : IDataExtensionSettings
7+
{
8+
[Required]
9+
public string FileName { get; set; } = null!;
10+
[Required]
11+
public string S3Region { get; set; } = null!;
12+
[Required]
13+
public string S3BucketName { get; set; } = null!;
14+
[Required]
15+
public string S3AccessKey { get; set; } = null!;
16+
[Required]
17+
public string S3SecretKey { get; set; } = null!;
18+
}
19+
}

‎Extensions/AwsS3/Cosmos.DataTransfer.AwsS3Storage/Cosmos.DataTransfer.AwsS3Storage.csproj

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
11+
<PackageReference Include="AWSSDK.S3" Version="3.7.104" />
12+
</ItemGroup>
13+
14+
<ItemGroup>
15+
<ProjectReference Include="..\..\..\Interfaces\Cosmos.DataTransfer.Interfaces\Cosmos.DataTransfer.Interfaces.csproj" />
16+
</ItemGroup>
17+
18+
</Project>

‎Extensions/AwsS3/Cosmos.DataTransfer.AwsS3Storage/S3Writer.cs

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using Amazon;
2+
using Amazon.S3;
3+
using Amazon.S3.Transfer;
4+
5+
namespace Cosmos.DataTransfer.AwsS3Storage
6+
{
7+
public static class S3Writer
8+
{
9+
private static IAmazonS3 s3Client;
10+
11+
public static void InitializeS3Client(string accessKey, string secretKey, string regionname)
12+
{
13+
RegionEndpoint region = RegionEndpoint.GetBySystemName(regionname);
14+
s3Client = new AmazonS3Client(accessKey, secretKey, region);
15+
}
16+
17+
public static async Task WriteToS3(string bucketName, Stream data, string filename, CancellationToken cancellationToken)
18+
{
19+
var ftu = new TransferUtility(s3Client);
20+
await ftu.UploadAsync(data, bucketName, filename, cancellationToken);
21+
}
22+
}
23+
}

‎Extensions/AwsS3/README.md

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# AWS S3 Extension (beta)
2+
3+
The AWS S3 extension provides writing of formatted files to AWS S3 buckets.
4+
5+
> **Note**: This is a Binary Storage Extension that is only used in combination with File Format extensions.
6+
7+
## Settings
8+
9+
Sink settings require all parameters shown below.
10+
11+
### Sink
12+
13+
```json
14+
{
15+
"FileName": "",
16+
"S3Region": "us-west-1",
17+
"S3BucketName": "",
18+
"S3AccessKey": "",
19+
"S3SecretKey": ""
20+
}
21+
```

0 commit comments

Comments
 (0)