Skip to content

Commit dff0877

Browse files
codingbanditcodingbandit
authored
Merge pull request AzureCosmosDB#19 from AzureCosmosDB/develop
Adding support for writing to Serverless accounts
2 parents 96c8ab3 + 52c5e48 commit dff0877

File tree

7 files changed

+49
-7
lines changed

7 files changed

+49
-7
lines changed

‎Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"profiles": {
3-
".Core": {
3+
"Cosmos.DataTransfer.Core": {
44
"commandName": "Project",
55
"commandLineArgs": ""
66
},

‎Core/Cosmos.DataTransfer.Core/appsettings.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
22
"Logging": {
33
"LogLevel": {
4-
"Default": "Information"
4+
"Default": "Information",
5+
"Microsoft.Hosting": "Warning"
56
}
67
},
78
"ExtensionsPath": null,

‎CosmosDbDataMigrationTool.sln

+15
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,26 @@ EndProject
1818
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.AzureTableAPIExtension", "Extensions\AzureTableAPI\Cosmos.DataTransfer.AzureTableAPIExtension\Cosmos.DataTransfer.AzureTableAPIExtension.csproj", "{CA03961C-4330-4ED9-963A-3B63FBC08ABB}"
1919
EndProject
2020
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Cosmos", "Cosmos", "{D0475879-F47F-4D12-9A43-812043CFFD22}"
21+
ProjectSection(SolutionItems) = preProject
22+
Extensions\Cosmos\README.md = Extensions\Cosmos\README.md
23+
EndProjectSection
2124
EndProject
2225
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Json", "Json", "{84E51D21-FA45-4CCB-94BB-6B6066749A4A}"
26+
ProjectSection(SolutionItems) = preProject
27+
Extensions\Json\README.md = Extensions\Json\README.md
28+
EndProjectSection
2329
EndProject
2430
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AzureTableAPI", "AzureTableAPI", "{CB41F880-D0A5-42C1-9571-479CF7E54D82}"
31+
ProjectSection(SolutionItems) = preProject
32+
Extensions\AzureTableAPI\README.md = Extensions\AzureTableAPI\README.md
33+
EndProjectSection
2534
EndProject
2635
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests", "Extensions\AzureTableAPI\Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests\Cosmos.DataTransfer.AzureTableAPIExtension.UnitTests.csproj", "{F31FADBA-BC78-4839-A9F5-8861695EC95E}"
2736
EndProject
2837
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Mongo", "Mongo", "{F18E789A-D32D-48D3-B75F-1196D7215F74}"
38+
ProjectSection(SolutionItems) = preProject
39+
Extensions\Mongo\README.md = Extensions\Mongo\README.md
40+
EndProjectSection
2941
EndProject
3042
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.MongoExtension", "Extensions\Mongo\Cosmos.DataTransfer.MongoExtension\Cosmos.DataTransfer.MongoExtension.csproj", "{F6EAC33B-9F7D-433B-9328-622FB8938C24}"
3143
EndProject
@@ -34,6 +46,9 @@ EndProject
3446
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.CosmosExtension.UnitTests", "Extensions\Cosmos\Cosmos.DataTransfer.CosmosExtension.UnitTests\Cosmos.DataTransfer.CosmosExtension.UnitTests.csproj", "{C7A3910D-A7F6-4767-9A7A-19CFA4CCB7A8}"
3547
EndProject
3648
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SqlServer", "SqlServer", "{2F075279-E8B0-4A6E-A8D9-2058B7DEC671}"
49+
ProjectSection(SolutionItems) = preProject
50+
Extensions\SqlServer\README.md = Extensions\SqlServer\README.md
51+
EndProjectSection
3752
EndProject
3853
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.SqlServerExtension", "Extensions\SqlServer\Cosmos.DataTransfer.SqlServerExtension\Cosmos.DataTransfer.SqlServerExtension.csproj", "{7A020621-77E6-4DD1-B230-50A46B4BB2B1}"
3954
EndProject

‎Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/Cosmos.DataTransfer.CosmosExtension.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
</PropertyGroup>
99

1010
<ItemGroup>
11-
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.31.0" />
11+
<PackageReference Include="Microsoft.Azure.Cosmos" Version="3.32.0" />
1212
<PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
1313
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="6.0.0" />
1414
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />

‎Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs

+21-2
Original file line numberDiff line numberDiff line change
@@ -68,11 +68,30 @@ public async Task WriteAsync(IAsyncEnumerable<IDataItem> dataItems, IConfigurati
6868
PartitionKeyPath = settings.PartitionKeyPath,
6969
};
7070

71-
ThroughputProperties throughputProperties = settings.UseAutoscaleForCreatedContainer
71+
ThroughputProperties? throughputProperties = settings.IsServerlessAccount
72+
? null
73+
: settings.UseAutoscaleForCreatedContainer
7274
? ThroughputProperties.CreateAutoscaleThroughput(settings.CreatedContainerMaxThroughput ?? 4000)
7375
: ThroughputProperties.CreateManualThroughput(settings.CreatedContainerMaxThroughput ?? 400);
7476

75-
Container? container = await database.CreateContainerIfNotExistsAsync(containerProperties, throughputProperties, cancellationToken: cancellationToken);
77+
Container? container;
78+
try
79+
{
80+
container = await database.CreateContainerIfNotExistsAsync(containerProperties, throughputProperties, cancellationToken: cancellationToken);
81+
}
82+
catch (CosmosException ex) when (ex.ResponseBody.Contains("not supported for serverless accounts", StringComparison.InvariantCultureIgnoreCase))
83+
{
84+
logger.LogWarning("Cosmos Serverless Account does not support throughput options. Creating Container {ContainerName} without those settings.", settings.Container);
85+
86+
// retry without throughput settings which are incompatible with serverless
87+
container = await database.CreateContainerIfNotExistsAsync(containerProperties, cancellationToken: cancellationToken);
88+
}
89+
90+
if (container == null)
91+
{
92+
logger.LogError("Failed to initialize Container {Container}", settings.Container);
93+
throw new Exception("Cosmos container unavailable for write");
94+
}
7695

7796
int addedCount = 0;
7897

‎Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSinkSettings.cs

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public class CosmosSinkSettings : IDataExtensionSettings
2121
public int InitialRetryDurationMs { get; set; } = 200;
2222
public int? CreatedContainerMaxThroughput { get; set; }
2323
public bool UseAutoscaleForCreatedContainer { get; set; } = true;
24+
public bool IsServerlessAccount { get; set; } = false;
2425
public DataWriteMode WriteMode { get; set; } = DataWriteMode.InsertStream;
2526
}
2627
}

‎Extensions/Cosmos/README.md

+8-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ Source supports an optional `IncludeMetadataFields` parameter (`false` by defaul
2626
}
2727
```
2828

29-
Sink requires an additional `PartitionKeyPath` parameter which is used when creating the container if it does not exist. It also supports an optional `RecreateContainer` parameter (`false` by default) to delete and then recreate the container to ensure only newly imported data is present. The optional `BatchSize` parameter (100 by default) sets the number of items to accumulate before inserting. The optional WriteMode parameter specifies the type of data write to use: InsertStream, Insert, UpsertStream, or Upsert.
29+
Sink requires an additional `PartitionKeyPath` parameter which is used when creating the container if it does not exist. It also supports an optional `RecreateContainer` parameter (`false` by default) to delete and then recreate the container to ensure only newly imported data is present. The optional `BatchSize` parameter (100 by default) sets the number of items to accumulate before inserting. The optional `WriteMode` parameter specifies the type of data write to use: `InsertStream`, `Insert`, `UpsertStream`, or `Upsert`. The `IsServerlessAccount` parameter specifies whether the target account uses Serverless instead of Provisioned throughput, which affects the way containers are created. Additional parameters allow changing the behavior of the Cosmos client appropriate to your environment.
3030

3131
### Sink
3232

@@ -38,6 +38,12 @@ Sink requires an additional `PartitionKeyPath` parameter which is used when crea
3838
"PartitionKeyPath":"/id",
3939
"RecreateContainer": false,
4040
"BatchSize": 100,
41-
"WriteMode": "InsertStream"
41+
"ConnectionMode": "Gateway",
42+
"MaxRetryCount": 5,
43+
"InitialRetryDurationMs": 200,
44+
"CreatedContainerMaxThroughput": 1000,
45+
"UseAutoscaleForCreatedContainer": true,
46+
"WriteMode": "InsertStream",
47+
"IsServerlessAccount": false
4248
}
4349
```

0 commit comments

Comments
 (0)