Skip to content

Commit 97f58b2

Browse files
bowencodebowencode
authored
Merge pull request #156 from stefanedwards/fix-csv-culture
fix: CSV extenstion uses invariant culture
2 parents 990b376 + 5fa1f18 commit 97f58b2

File tree

4 files changed

+180
-7
lines changed

4 files changed

+180
-7
lines changed

‎Extensions/Csv/Cosmos.DataTransfer.CsvExtension.UnitTests/CsvWriterSettingsTests.cs

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
using System.Globalization;
2+
using Cosmos.DataTransfer.JsonExtension.UnitTests;
3+
using Microsoft.Extensions.Logging.Abstractions;
4+
using System.ComponentModel.DataAnnotations;
5+
using Cosmos.DataTransfer.CsvExtension.Settings;
6+
using Cosmos.DataTransfer.Interfaces;
7+
using Cosmos.DataTransfer.JsonExtension;
8+
9+
namespace Cosmos.DataTransfer.CsvExtension.UnitTests;
10+
11+
[TestClass]
12+
public class CsvWriterSettingsTests
13+
{
14+
15+
16+
[TestMethod]
17+
public void TestDefault() {
18+
var settings = new CsvWriterSettings() { };
19+
20+
Assert.AreEqual(CultureInfo.InvariantCulture, settings.GetCultureInfo());
21+
Assert.AreEqual(0, settings.Validate(new ValidationContext(this)).Count());
22+
}
23+
24+
[TestMethod]
25+
[DataRow("invariant")]
26+
[DataRow("Invariant")]
27+
[DataRow("invariantCulture")]
28+
[DataRow("invariantculture")]
29+
public void TestInvariantCulture(string culture) {
30+
var settings = new CsvWriterSettings() {
31+
Culture = culture
32+
};
33+
Assert.AreEqual(CultureInfo.InvariantCulture, settings.GetCultureInfo());
34+
Assert.AreEqual(0, settings.Validate(new ValidationContext(this)).Count());
35+
}
36+
37+
[TestMethod]
38+
[DataRow("current")]
39+
[DataRow("Current")]
40+
[DataRow("currentCultuRE")]
41+
[DataRow("currentCulture")]
42+
public void TestCurrentCulture(string culture) {
43+
var settings = new CsvWriterSettings() {
44+
Culture = culture
45+
};
46+
Assert.AreEqual(CultureInfo.CurrentCulture, settings.GetCultureInfo());
47+
Assert.AreEqual(0, settings.Validate(new ValidationContext(this)).Count());
48+
}
49+
50+
[TestMethod]
51+
public void TestCurrentCultureByName() {
52+
if (string.IsNullOrEmpty(CultureInfo.CurrentCulture.Name))
53+
{
54+
Assert.Inconclusive("Current culture name in executing environment is empty.");
55+
}
56+
57+
var settings = new CsvWriterSettings() {
58+
Culture = CultureInfo.CurrentCulture.Name
59+
};
60+
Assert.AreEqual(CultureInfo.CurrentCulture, settings.GetCultureInfo());
61+
Assert.AreEqual(0, settings.Validate(new ValidationContext(this)).Count());
62+
}
63+
64+
[TestMethod]
65+
public void TestCultureFails() {
66+
var settings = new CsvWriterSettings() {
67+
Culture = "not a culture"
68+
};
69+
var results = settings.Validate(new ValidationContext(this)).ToArray();
70+
Assert.AreEqual(1, results.Count());
71+
Assert.AreEqual("Could not find CultureInfo `not a culture` on this system.", results.First().ErrorMessage);
72+
}
73+
74+
[TestMethod]
75+
public void TestCultureMissing() {
76+
var settings = new CsvWriterSettings() {
77+
Culture = ""
78+
};
79+
var results = settings.Validate(new ValidationContext(this)).ToArray();
80+
Assert.AreEqual(1, results.Count());
81+
Assert.AreEqual("Culture missing.", results.First().ErrorMessage);
82+
}
83+
84+
[TestMethod]
85+
public void TestCultureNull()
86+
{
87+
var settings = new CsvWriterSettings()
88+
{
89+
Culture = null
90+
};
91+
var results = settings.Validate(new ValidationContext(this)).ToArray();
92+
Assert.AreEqual(1, results.Count());
93+
Assert.AreEqual("Culture missing.", results.First().ErrorMessage);
94+
}
95+
96+
[TestMethod]
97+
public async Task TestDanishCulture() {
98+
var outputFile = Path.GetTempFileName();
99+
var config = TestHelpers.CreateConfig(new Dictionary<string, string>
100+
{
101+
{ "FilePath", outputFile },
102+
{ "IncludeHeader", "false" },
103+
{ "Culture", "da-DK" },
104+
{ "Delimiter", ";" }
105+
});
106+
107+
var data = new List<DictionaryDataItem>
108+
{
109+
new(new Dictionary<string, object?>
110+
{
111+
{ "Value", 1.2 }
112+
})
113+
};
114+
115+
var sink = new CsvFileSink();
116+
117+
await sink.WriteAsync(data.ToAsyncEnumerable(), config, new JsonFileSource(), NullLogger.Instance);
118+
var result = await File.ReadAllTextAsync(outputFile);
119+
Assert.AreEqual("1,2", result);
120+
}
121+
}

‎Extensions/Csv/Cosmos.DataTransfer.CsvExtension/CsvFormatWriter.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ public async Task FormatDataAsync(IAsyncEnumerable<IDataItem> dataItems, Stream
2222
settings.Validate();
2323

2424
await using var textWriter = new StreamWriter(target, leaveOpen: true);
25-
await using var writer = new CsvWriter(textWriter, new CsvConfiguration(CultureInfo.InvariantCulture)
25+
await using var writer = new CsvWriter(textWriter, new CsvConfiguration(settings.GetCultureInfo())
2626
{
27-
Delimiter = settings.Delimiter,
27+
Delimiter = settings.Delimiter ?? ",",
2828
HasHeaderRecord = settings.IncludeHeader,
2929
});
3030

@@ -49,7 +49,7 @@ public async Task FormatDataAsync(IAsyncEnumerable<IDataItem> dataItems, Stream
4949

5050
foreach (string field in item.GetFieldNames())
5151
{
52-
writer.WriteField(item.GetValue(field)?.ToString());
52+
writer.WriteField(item.GetValue(field));
5353
}
5454

5555
firstRecord = false;

‎Extensions/Csv/Cosmos.DataTransfer.CsvExtension/Settings/CsvWriterSettings.cs

+44-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,50 @@
1-
using Cosmos.DataTransfer.Interfaces;
1+
using System.Globalization;
2+
using Cosmos.DataTransfer.Interfaces;
3+
using System.ComponentModel.DataAnnotations;
24

35
namespace Cosmos.DataTransfer.CsvExtension.Settings;
46

5-
public class CsvWriterSettings : IDataExtensionSettings
7+
public class CsvWriterSettings : IDataExtensionSettings, IValidatableObject
68
{
79
public bool IncludeHeader { get; set; } = true;
8-
public string Delimiter { get; set; } = ",";
10+
public string? Delimiter { get; set; } = ",";
11+
public string? Culture { get; set; } = "InvariantCulture";
12+
public CultureInfo GetCultureInfo() {
13+
switch (this.Culture?.ToLower())
14+
{
15+
case "invariant":
16+
case "invariantculture":
17+
return CultureInfo.InvariantCulture;
18+
case "current":
19+
case "currentculture":
20+
return CultureInfo.CurrentCulture;
21+
case "":
22+
case null:
23+
throw new ArgumentNullException();
24+
default: return CultureInfo.GetCultureInfo(this.Culture!);
25+
}
26+
}
27+
28+
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
29+
{
30+
ValidationResult? result = null;
31+
try {
32+
_ = this.GetCultureInfo();
33+
} catch (CultureNotFoundException) {
34+
result = new ValidationResult(
35+
$"Could not find CultureInfo `{this.Culture}` on this system.",
36+
new string[] { "Culture" }
37+
);
38+
} catch (ArgumentNullException) {
39+
result = new ValidationResult(
40+
$"Culture missing.",
41+
new string[] { "Culture" }
42+
);
43+
}
44+
45+
46+
if (result != null) {
47+
yield return result;
48+
}
49+
}
950
}

‎Extensions/Csv/README.md

+12-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ See storage extension documentation for any storage specific settings needed ([e
2424

2525
Source supports an optional `Delimiter` parameter (`,` by default) and an optional `HasHeader` parameter (`true` by default). For files without a header, column names will be generated based on the `ColumnNameFormat` setting, which uses a default value of `column_{0}` to produce columns `column_0`, `column_1`, etc.
2626

27+
2728
```json
2829
{
2930
"Delimiter": ",",
@@ -35,9 +36,19 @@ Source supports an optional `Delimiter` parameter (`,` by default) and an option
3536

3637
Sink supports an optional `Delimiter` parameter (`,` by default) and an optional `IncludeHeader` parameter (`true` by default) to add a leading row of column names.
3738

39+
Formatting options, or locale, can be set with an optional `Culture` setting (`"InvariantCulture"` by default).
40+
This specifies how e.g., numbers and dates are formatted according to a specific culture.
41+
Set to `"InvariantCulture"` to use the system's or process' current locale setting
42+
(see [CultureInfo.CurrentCulture](https://learn.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.currentculture)),
43+
or e.g., `"en"`, `"en-GB"`, or `"en-US"` for English standards (period, `.`, as decimal separator and other regional standards),
44+
"da-DK" for Danish (comma, `,`, as decimal separator), etc.
45+
Note, if using a culture with comma as decimal separator, specify a different delimiter (e.g., semi-colon, `;`), else all numbers
46+
will be written enclosed with quotes.
47+
3848
```json
3949
{
4050
"Delimiter": ",",
41-
"IncludeHeader": true
51+
"IncludeHeader": true,
52+
"Culture": "Invariant"
4253
}
4354
```

0 commit comments

Comments
 (0)