Skip to content

Commit 4bf9049

Browse files
GuillaumeVadeBeGuillaumeVadeBe
authored
Merge branch 'AzureCosmosDB:main' into main
2 parents 5dd9523 + 574cf77 commit 4bf9049

File tree

38 files changed

+432
-156
lines changed

38 files changed

+432
-156
lines changed

‎.github/dependabot.yml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# To get started with Dependabot version updates, you'll need to specify which
2+
# package ecosystems to update and where the package manifests are located.
3+
# Please see the documentation for all configuration options:
4+
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
5+
6+
version: 2
7+
updates:
8+
- package-ecosystem: "" # See documentation for possible values
9+
directory: "/" # Location of package manifests
10+
schedule:
11+
interval: "weekly"

‎Core/Cosmos.DataTransfer.Core/Cosmos.DataTransfer.Core.csproj

+4-3
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919
</PropertyGroup>
2020

2121
<ItemGroup>
22-
<PackageReference Include="Azure.Core" Version="1.36.0" />
23-
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.0" />
22+
<PackageReference Include="Azure.Core" Version="1.40.0" />
23+
<PackageReference Include="Azure.Identity" Version="1.12.0" />
24+
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
2425
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="6.0.1" />
2526
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
2627
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
2728
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
2829
<PackageReference Include="System.CommandLine.Hosting" Version="0.4.0-alpha.22272.1" />
29-
<PackageReference Include="System.ComponentModel.Composition" Version="6.0.0" />
30+
<PackageReference Include="System.ComponentModel.Composition" Version="7.0.0" />
3031
<PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
3132
</ItemGroup>
3233

‎CosmosDbDataMigrationTool.sln

+3
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ EndProject
6464
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.Common", "Interfaces\Cosmos.DataTransfer.Common\Cosmos.DataTransfer.Common.csproj", "{0FAD9D89-2E41-4D65-8440-5C01885D9292}"
6565
EndProject
6666
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AzureBlob", "AzureBlob", "{9627A42A-BEB0-4A39-B49C-C3C6D54E705A}"
67+
ProjectSection(SolutionItems) = preProject
68+
Extensions\AzureBlob\README.md = Extensions\AzureBlob\README.md
69+
EndProjectSection
6770
EndProject
6871
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AwsS3", "AwsS3", "{502197E4-F554-4B5B-9235-FBFE7E49EBEF}"
6972
EndProject

‎ExampleConfigs.md

+113-80
Original file line numberDiff line numberDiff line change
@@ -1,123 +1,156 @@
11
# Example `migrationsettings.json` Files
22

33
## JSON to Cosmos-NoSQL
4+
45
```json
56
{
67
"Source": "json",
78
"Sink": "cosmos-nosql",
8-
"SourceSettings": {
9-
"FilePath": "https://mytestfiles.local/sales-data.json"
10-
},
11-
"SinkSettings": {
12-
"ConnectionString": "AccountEndpoint=https://...",
13-
"Database": "myDb",
14-
"Container": "myContainer",
15-
"PartitionKeyPath": "/id",
16-
"RecreateContainer": true,
17-
"WriteMode": "Insert",
18-
"CreatedContainerMaxThroughput": 5000,
19-
"IsServerlessAccount": false
20-
}
9+
"SourceSettings": {
10+
"FilePath": "https://mytestfiles.local/sales-data.json"
11+
},
12+
"SinkSettings": {
13+
"ConnectionString": "AccountEndpoint=https://...",
14+
"Database": "myDb",
15+
"Container": "myContainer",
16+
"PartitionKeyPath": "/id",
17+
"RecreateContainer": true,
18+
"WriteMode": "Insert",
19+
"CreatedContainerMaxThroughput": 5000,
20+
"IsServerlessAccount": false
21+
}
2122
}
2223
```
2324

2425
## Cosmos-NoSQL to JSON
26+
2527
```json
2628
{
2729
"Source": "Cosmos-NoSql",
2830
"Sink": "JSON",
29-
"SourceSettings":
30-
{
31-
"ConnectionString": "AccountEndpoint=https://...",
32-
"Database":"cosmicworks",
33-
"Container":"customers",
34-
"IncludeMetadataFields": true
35-
},
36-
"SinkSettings":
37-
{
38-
"FilePath": "c:\\data\\cosmicworks\\customers.json",
39-
"Indented": true
40-
}
31+
"SourceSettings":
32+
{
33+
"ConnectionString": "AccountEndpoint=https://...",
34+
"Database":"cosmicworks",
35+
"Container":"customers",
36+
"IncludeMetadataFields": true
37+
},
38+
"SinkSettings":
39+
{
40+
"FilePath": "c:\\data\\cosmicworks\\customers.json",
41+
"Indented": true
42+
}
4143
}
4244
```
4345

4446
## MongoDB to Cosmos-NoSQL
47+
4548
```json
4649
{
4750
"Source": "mongodb",
4851
"Sink": "cosmos-nosql",
49-
"SourceSettings": {
50-
"ConnectionString": "mongodb://...",
51-
"DatabaseName": "sales",
52-
"Collection": "person"
53-
},
54-
"SinkSettings": {
55-
"ConnectionString": "AccountEndpoint=https://...",
56-
"Database": "users",
57-
"Container": "migrated",
58-
"PartitionKeyPath": "/id",
52+
"SourceSettings": {
53+
"ConnectionString": "mongodb://...",
54+
"DatabaseName": "sales",
55+
"Collection": "person"
56+
},
57+
"SinkSettings": {
58+
"ConnectionString": "AccountEndpoint=https://...",
59+
"Database": "users",
60+
"Container": "migrated",
61+
"PartitionKeyPath": "/id",
5962
"ConnectionMode": "Direct",
60-
"WriteMode": "UpsertStream",
61-
"CreatedContainerMaxThroughput": 8000,
62-
"UseAutoscaleForCreatedContainer": false
63-
}
63+
"WriteMode": "UpsertStream",
64+
"CreatedContainerMaxThroughput": 8000,
65+
"UseAutoscaleForCreatedContainer": false
66+
}
6467
}
6568
```
6669

6770
## SqlServer to AzureTableAPI
71+
6872
```json
6973
{
7074
"Source": "SqlServer",
7175
"Sink": "AzureTableApi",
72-
"SourceSettings": {
73-
"ConnectionString": "Server=...",
74-
"QueryText": "SELECT Id, Date, Amount FROM dbo.Payments WHERE Status = 'open'"
75-
},
76-
"SinkSettings": {
77-
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=...",
78-
"Table": "payments",
79-
"RowKeyFieldName": "Id"
80-
}
76+
"SourceSettings": {
77+
"ConnectionString": "Server=...",
78+
"QueryText": "SELECT Id, Date, Amount FROM dbo.Payments WHERE Status = 'open'"
79+
},
80+
"SinkSettings": {
81+
"ConnectionString": "DefaultEndpointsProtocol=https;AccountName=...",
82+
"Table": "payments",
83+
"RowKeyFieldName": "Id"
84+
}
8185
}
8286
```
8387

8488
## Cosmos-NoSQL to SqlServer
89+
8590
```json
8691
{
8792
"Source": "cosmos-nosql",
8893
"Sink": "sqlserver",
89-
"SourceSettings":
90-
{
91-
"ConnectionString": "AccountEndpoint=https://...",
92-
"Database":"operations",
93-
"Container":"alerts",
94-
"PartitionKeyValue": "jan",
94+
"SourceSettings":
95+
{
96+
"ConnectionString": "AccountEndpoint=https://...",
97+
"Database":"operations",
98+
"Container":"alerts",
99+
"PartitionKeyValue": "jan",
95100
"Query": "SELECT a.name, a.description, a.count, a.id, a.isSet FROM a"
96-
},
97-
"SinkSettings":
98-
{
99-
"ConnectionString": "Server=...",
100-
"TableName": "Import",
101-
"ColumnMappings": [
102-
{
103-
"ColumnName": "Name"
104-
},
105-
{
106-
"ColumnName": "Description"
107-
},
108-
{
109-
"ColumnName": "Count",
110-
"SourceFieldName": "number"
111-
},
112-
{
113-
"ColumnName": "Id"
114-
},
115-
{
116-
"ColumnName": "IsSet",
117-
"AllowNull": false,
118-
"DefaultValue": false
119-
}
120-
]
121-
}
101+
},
102+
"SinkSettings":
103+
{
104+
"ConnectionString": "Server=...",
105+
"TableName": "Import",
106+
"ColumnMappings": [
107+
{
108+
"ColumnName": "Name"
109+
},
110+
{
111+
"ColumnName": "Description"
112+
},
113+
{
114+
"ColumnName": "Count",
115+
"SourceFieldName": "number"
116+
},
117+
{
118+
"ColumnName": "Id"
119+
},
120+
{
121+
"ColumnName": "IsSet",
122+
"AllowNull": false,
123+
"DefaultValue": false
124+
}
125+
]
126+
}
127+
}
128+
```
129+
130+
## Cosmos-NoSQL to Json-AzureBlob (Using RBAC)
131+
132+
```json
133+
{
134+
"Source": "Cosmos-nosql",
135+
"Sink": "Json-AzureBlob",
136+
"SourceSettings": {
137+
"UseRbacAuth": true,
138+
"Database": "operations",
139+
"Container": "alerts",
140+
"PartitionKeyValue": "jan",
141+
"AccountEndpoint": "https://<databaseaccount>.documents.azure.com",
142+
"EnableInteractiveCredentials": true,
143+
"IncludeMetadataFields": false,
144+
"Query": "SELECT a.name, a.description, a.count, a.id, a.isSet FROM a"
145+
},
146+
"SinkSettings": {
147+
"UseRbacAuth": true,
148+
"ContainerName": "operations-archive",
149+
"AccountEndpoint": "https://<storage-account>.blob.core.windows.net",
150+
"EnableInteractiveCredentials": true,
151+
"BlobName": "jan-alerts"
152+
},
153+
"Operations": [
154+
]
122155
}
123156
```

‎Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs

+25-5
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
using Azure.Storage.Blobs;
1+
using Azure.Identity;
2+
using Azure.Storage.Blobs;
3+
using Azure.Storage.Blobs.Models;
4+
using Azure.Storage.Blobs.Specialized;
25
using Cosmos.DataTransfer.Interfaces;
36
using Microsoft.Extensions.Configuration;
47
using Microsoft.Extensions.Logging;
5-
using Azure.Storage.Blobs.Specialized;
6-
using Azure.Storage.Blobs.Models;
78

89
namespace Cosmos.DataTransfer.AzureBlobStorage
910
{
@@ -14,12 +15,31 @@ public async Task WriteToTargetAsync(Func<Stream, Task> writeToStream, IConfigur
1415
var settings = config.Get<AzureBlobSinkSettings>();
1516
settings.Validate();
1617

17-
logger.LogInformation("Saving file '{File}' to Azure Blob Container '{ContainerName}'", settings.BlobName, settings.ContainerName);
18+
BlobContainerClient account;
19+
if (settings.UseRbacAuth)
20+
{
21+
logger.LogInformation("Connecting to Storage account {AccountEndpoint} using {UseRbacAuth} with {EnableInteractiveCredentials}'", settings.AccountEndpoint, nameof(AzureBlobSourceSettings.UseRbacAuth), nameof(AzureBlobSourceSettings.EnableInteractiveCredentials));
22+
23+
var credential = new DefaultAzureCredential(includeInteractiveCredentials: settings.EnableInteractiveCredentials);
24+
#pragma warning disable CS8604 // Validate above ensures AccountEndpoint is not null
25+
var baseUri = new Uri(settings.AccountEndpoint);
26+
var blobContainerUri = new Uri(baseUri, settings.ContainerName);
27+
#pragma warning restore CS8604 // Restore warning
28+
29+
account = new BlobContainerClient(blobContainerUri, credential);
30+
}
31+
else
32+
{
33+
logger.LogInformation("Connecting to Storage account using {ConnectionString}'", nameof(AzureBlobSourceSettings.ConnectionString));
34+
35+
account = new BlobContainerClient(settings.ConnectionString, settings.ContainerName);
36+
}
1837

19-
var account = new BlobContainerClient(settings.ConnectionString, settings.ContainerName);
2038
await account.CreateIfNotExistsAsync(cancellationToken: cancellationToken);
2139
var blob = account.GetBlockBlobClient(settings.BlobName);
2240

41+
logger.LogInformation("Saving file '{File}' to Azure Blob Container '{ContainerName}'", settings.BlobName, settings.ContainerName);
42+
2343
await using var blobStream = await blob.OpenWriteAsync(true, new BlockBlobOpenWriteOptions
2444
{
2545
BufferSize = settings.MaxBlockSizeinKB * 1024L,

‎Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs

+25-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
using System.Runtime.CompilerServices;
1+
using Azure.Identity;
22
using Azure.Storage.Blobs;
3+
using Azure.Storage.Blobs.Models;
4+
using Azure.Storage.Blobs.Specialized;
35
using Cosmos.DataTransfer.Interfaces;
46
using Microsoft.Extensions.Configuration;
57
using Microsoft.Extensions.Logging;
6-
using Azure.Storage.Blobs.Specialized;
7-
using Azure.Storage.Blobs.Models;
8+
using System.Runtime.CompilerServices;
89

910
namespace Cosmos.DataTransfer.AzureBlobStorage;
1011

@@ -15,14 +16,33 @@ public class AzureBlobDataSource : IComposableDataSource
1516
var settings = config.Get<AzureBlobSourceSettings>();
1617
settings.Validate();
1718

18-
logger.LogInformation("Reading file '{File}' from Azure Blob Container '{ContainerName}'", settings.BlobName, settings.ContainerName);
19+
BlobContainerClient account;
20+
if (settings.UseRbacAuth)
21+
{
22+
logger.LogInformation("Connecting to Storage account {AccountEndpoint} using {UseRbacAuth} with {EnableInteractiveCredentials}'", settings.AccountEndpoint, nameof(AzureBlobSourceSettings.UseRbacAuth), nameof(AzureBlobSourceSettings.EnableInteractiveCredentials));
1923

20-
var account = new BlobContainerClient(settings.ConnectionString, settings.ContainerName);
24+
var credential = new DefaultAzureCredential(includeInteractiveCredentials: settings.EnableInteractiveCredentials);
25+
#pragma warning disable CS8604 // Validate above ensures AccountEndpoint is not null
26+
var baseUri = new Uri(settings.AccountEndpoint);
27+
var blobContainerUri = new Uri(baseUri, settings.ContainerName);
28+
#pragma warning restore CS8604 // Restore warning
29+
30+
account = new BlobContainerClient(blobContainerUri, credential);
31+
}
32+
else
33+
{
34+
logger.LogInformation("Connecting to Storage account using {ConnectionString}'", nameof(AzureBlobSourceSettings.ConnectionString));
35+
36+
account = new BlobContainerClient(settings.ConnectionString, settings.ContainerName);
37+
}
38+
2139
var blob = account.GetBlockBlobClient(settings.BlobName);
2240
var existsResponse = await blob.ExistsAsync(cancellationToken: cancellationToken);
2341
if (!existsResponse)
2442
yield break;
2543

44+
logger.LogInformation("Reading file '{File}' from Azure Blob Container '{ContainerName}'", settings.BlobName, settings.ContainerName);
45+
2646
var readStream = await blob.OpenReadAsync(new BlobOpenReadOptions(false)
2747
{
2848
BufferSize = settings.ReadBufferSizeInKB,

0 commit comments

Comments
 (0)