Skip to content

Commit 486c5b7

Browse files
bowencodebowencode
authored
Merge pull request #59 from AzureCosmosDB/develop
Adding Settings command to generate extension manifest
2 parents f8a331d + a94dc96 commit 486c5b7

File tree

54 files changed

+659
-30
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+659
-30
lines changed

‎Core/Cosmos.DataTransfer.Core.UnitTests/SettingsCommandTests.cs

+92
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
using Cosmos.DataTransfer.Interfaces;
2+
using Cosmos.DataTransfer.Interfaces.Manifest;
3+
using Microsoft.Extensions.Logging.Abstractions;
4+
using Microsoft.VisualStudio.TestTools.UnitTesting;
5+
using Moq;
6+
using System;
7+
using System.Collections.Generic;
8+
using System.CommandLine;
9+
using System.CommandLine.Invocation;
10+
using System.ComponentModel.DataAnnotations;
11+
using System.Text;
12+
using System.Text.Json;
13+
using System.Text.Json.Serialization;
14+
15+
namespace Cosmos.DataTransfer.Core.UnitTests
16+
{
17+
[TestClass]
18+
public class SettingsCommandTests
19+
{
20+
[TestMethod]
21+
public void Invoke_ForTestExtension_ProducesValidSettingsJson()
22+
{
23+
var command = new SettingsCommand();
24+
const string source = "testSource";
25+
var loader = new Mock<IExtensionManifestBuilder>();
26+
var sourceExtension = new Mock<IDataSourceExtensionWithSettings>();
27+
sourceExtension.SetupGet(ds => ds.DisplayName).Returns(source);
28+
sourceExtension.Setup(ds => ds.GetSettings()).Returns(new List<IDataExtensionSettings>
29+
{
30+
new MockExtensionSettings(),
31+
new MockExtensionSettings2()
32+
});
33+
loader
34+
.Setup(l => l.GetSources())
35+
.Returns(new List<IDataSourceExtension> { sourceExtension.Object });
36+
37+
var writer = new Mock<IRawOutputWriter>();
38+
var outputLines = new List<string>();
39+
writer.Setup(w => w.WriteLine(It.IsAny<string>())).Callback<string>(s => outputLines.Add(s));
40+
var handler = new SettingsCommand.CommandHandler(loader.Object, writer.Object, NullLogger<SettingsCommand.CommandHandler>.Instance)
41+
{
42+
Source = true,
43+
Extension = source
44+
};
45+
46+
var parseResult = new SettingsCommand().Parse(Array.Empty<string>());
47+
var result = handler.Invoke(new InvocationContext(parseResult));
48+
Assert.AreEqual(0, result);
49+
50+
bool jsonStarted = false;
51+
var stringBuilder = new StringBuilder();
52+
foreach (string item in outputLines)
53+
{
54+
if (item == "<<<")
55+
jsonStarted = true;
56+
else if (item == ">>>")
57+
jsonStarted = false;
58+
else if (jsonStarted)
59+
stringBuilder.AppendLine(item);
60+
}
61+
62+
var options = new JsonSerializerOptions
63+
{
64+
Converters = { new JsonStringEnumConverter() },
65+
WriteIndented = true
66+
};
67+
var fullJson = stringBuilder.ToString().Trim();
68+
var parsed = JsonSerializer.Deserialize<List<ExtensionSettingProperty>>(fullJson, options);
69+
var parsedJson = JsonSerializer.Serialize<List<ExtensionSettingProperty>>(parsed, options);
70+
71+
Assert.AreEqual(fullJson, parsedJson);
72+
}
73+
}
74+
75+
public class MockExtensionSettings : IDataExtensionSettings
76+
{
77+
[Required]
78+
public string Name { get; set; } = null!;
79+
public string? Description { get; set; }
80+
public int Count { get; set; } = 99;
81+
public int? MaxValue { get; set; }
82+
public double? Avg { get; set; }
83+
public bool Enabled { get; set; } = true;
84+
public JsonCommentHandling TestEnum { get; set; }
85+
}
86+
87+
public class MockExtensionSettings2 : IDataExtensionSettings
88+
{
89+
[Required]
90+
public string? AnotherProperty { get; set; }
91+
}
92+
}

‎Core/Cosmos.DataTransfer.Core/ExtensionManifestBuilder.cs

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
using Cosmos.DataTransfer.Interfaces;
2+
using Cosmos.DataTransfer.Interfaces.Manifest;
3+
using Microsoft.Extensions.Logging;
4+
using System;
5+
using System.ComponentModel.Composition.Hosting;
6+
using System.Reflection;
7+
8+
namespace Cosmos.DataTransfer.Core
9+
{
10+
public class ExtensionManifestBuilder : IExtensionManifestBuilder
11+
{
12+
private readonly ILogger _logger;
13+
private readonly IExtensionLoader _extensionLoader;
14+
15+
public ExtensionManifestBuilder(IExtensionLoader extensionLoader, ILogger<ExtensionManifestBuilder> logger)
16+
{
17+
_extensionLoader = extensionLoader;
18+
_logger = logger;
19+
}
20+
21+
public List<IDataSourceExtension> GetSources()
22+
{
23+
string extensionsPath = _extensionLoader.GetExtensionFolderPath();
24+
CompositionContainer container = _extensionLoader.BuildExtensionCatalog(extensionsPath);
25+
26+
return _extensionLoader.LoadExtensions<IDataSourceExtension>(container);
27+
}
28+
29+
public List<IDataSinkExtension> GetSinks()
30+
{
31+
string extensionsPath = _extensionLoader.GetExtensionFolderPath();
32+
CompositionContainer container = _extensionLoader.BuildExtensionCatalog(extensionsPath);
33+
34+
return _extensionLoader.LoadExtensions<IDataSinkExtension>(container);
35+
}
36+
37+
public ExtensionManifest BuildManifest(ExtensionDirection direction)
38+
{
39+
var extensions = new List<IDataTransferExtension>();
40+
if (direction == ExtensionDirection.Source)
41+
{
42+
extensions.AddRange(GetSources());
43+
}
44+
else
45+
{
46+
extensions.AddRange(GetSinks());
47+
}
48+
var manifest = new ExtensionManifest(extensions
49+
.Select(e => new ExtensionManifestItem(e.DisplayName,
50+
direction,
51+
GetExtensionSettings(e as IExtensionWithSettings))).ToList());
52+
return manifest;
53+
}
54+
55+
public List<ExtensionSettingProperty> GetExtensionSettings(IExtensionWithSettings? extension)
56+
{
57+
var allProperties = new List<ExtensionSettingProperty>();
58+
if (extension != null)
59+
{
60+
var allSettings = extension.GetSettings();
61+
foreach (IDataExtensionSettings settings in allSettings)
62+
{
63+
var settingsType = settings.GetType();
64+
65+
var props = settingsType.GetProperties();
66+
foreach (PropertyInfo propertyInfo in props)
67+
{
68+
var defaultValue = propertyInfo.GetValue(settings);
69+
var settingProperty = new ExtensionSettingProperty(propertyInfo.Name, GetPropertyType(propertyInfo.PropertyType))
70+
{
71+
IsRequired = propertyInfo.GetCustomAttribute<System.ComponentModel.DataAnnotations.RequiredAttribute>() is not null,
72+
DefaultValue = defaultValue,
73+
IsSensitive = propertyInfo.GetCustomAttribute<SensitiveValueAttribute>() is not null
74+
};
75+
if (settingProperty.Type == PropertyType.Enum)
76+
{
77+
settingProperty.ValidValues.AddRange(GetPropertyEnumValues(propertyInfo, settings));
78+
}
79+
allProperties.Add(settingProperty);
80+
}
81+
}
82+
}
83+
84+
return allProperties;
85+
}
86+
87+
private IEnumerable<string> GetPropertyEnumValues(PropertyInfo propertyInfo, IDataExtensionSettings settings)
88+
{
89+
if (propertyInfo.PropertyType.IsEnum)
90+
{
91+
return Enum.GetNames(propertyInfo.PropertyType);
92+
}
93+
94+
return Enumerable.Empty<string>();
95+
}
96+
97+
private static PropertyType GetPropertyType(Type type)
98+
{
99+
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>))
100+
{
101+
var genericType = type.GetGenericArguments().FirstOrDefault();
102+
if (genericType != null)
103+
type = genericType;
104+
}
105+
106+
if (type == typeof(byte) || type == typeof(short) || type == typeof(ushort) ||
107+
type == typeof(int) || type == typeof(uint) ||
108+
type == typeof(long) || type == typeof(ulong))
109+
{
110+
return PropertyType.Int;
111+
}
112+
113+
if (type == typeof(double) || type == typeof(float) || type == typeof(decimal))
114+
{
115+
return PropertyType.Float;
116+
}
117+
118+
if (type == typeof(bool))
119+
{
120+
return PropertyType.Boolean;
121+
}
122+
123+
if (type == typeof(DateTime) || type == typeof(DateTimeOffset))
124+
{
125+
return PropertyType.DateTime;
126+
}
127+
128+
if (type.IsEnum)
129+
{
130+
return PropertyType.Enum;
131+
}
132+
133+
return PropertyType.String;
134+
}
135+
}
136+
}

‎Core/Cosmos.DataTransfer.Core/IExtensionManifestBuilder.cs

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using Cosmos.DataTransfer.Interfaces;
2+
using Cosmos.DataTransfer.Interfaces.Manifest;
3+
using System;
4+
5+
namespace Cosmos.DataTransfer.Core
6+
{
7+
public interface IExtensionManifestBuilder
8+
{
9+
ExtensionManifest BuildManifest(ExtensionDirection direction);
10+
List<ExtensionSettingProperty> GetExtensionSettings(IExtensionWithSettings? extension);
11+
List<IDataSinkExtension> GetSinks();
12+
List<IDataSourceExtension> GetSources();
13+
}
14+
}

‎Core/Cosmos.DataTransfer.Core/ListCommand.cs

+33-6
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,31 @@ public class ListCommand : Command
1111
public ListCommand()
1212
: base("list", "Loads and lists all available extensions")
1313
{
14+
AddListOptions(this);
15+
}
16+
17+
public static void AddListOptions(Command command)
18+
{
19+
var sourcesOption = new Option<bool?>(
20+
aliases: new[] { "--sources" },
21+
description: "True to include source names");
22+
23+
var sinksOption = new Option<bool?>(
24+
aliases: new[] { "--sinks" },
25+
description: "True to include sink names");
26+
27+
command.AddOption(sourcesOption);
28+
command.AddOption(sinksOption);
1429
}
1530

1631
public class CommandHandler : ICommandHandler
1732
{
1833
private readonly ILogger<CommandHandler> _logger;
1934
private readonly IExtensionLoader _extensionLoader;
2035

36+
public bool? Sources { get; set; }
37+
public bool? Sinks { get; set; }
38+
2139
public CommandHandler(IExtensionLoader extensionLoader, ILogger<CommandHandler> logger)
2240
{
2341
_logger = logger;
@@ -32,16 +50,25 @@ public int Invoke(InvocationContext context)
3250
var sources = _extensionLoader.LoadExtensions<IDataSourceExtension>(container);
3351
var sinks = _extensionLoader.LoadExtensions<IDataSinkExtension>(container);
3452

35-
Console.WriteLine($"{sources.Count} Source Extensions");
36-
foreach (var extension in sources)
53+
bool showSources = Sources ?? (Sources == null && Sinks == null);
54+
bool showSinks = Sinks ?? (Sources == null && Sinks == null);
55+
56+
if (showSources)
3757
{
38-
Console.WriteLine($"\t{extension.DisplayName}");
58+
Console.WriteLine($"{sources.Count} Source Extensions");
59+
foreach (var extension in sources)
60+
{
61+
Console.WriteLine($"\t{extension.DisplayName}");
62+
}
3963
}
4064

41-
Console.WriteLine($"{sinks.Count} Sink Extensions");
42-
foreach (var extension in sinks)
65+
if (showSinks)
4366
{
44-
Console.WriteLine($"\t{extension.DisplayName}");
67+
Console.WriteLine($"{sinks.Count} Sink Extensions");
68+
foreach (var extension in sinks)
69+
{
70+
Console.WriteLine($"\t{extension.DisplayName}");
71+
}
4572
}
4673

4774
return 0;

‎Core/Cosmos.DataTransfer.Core/Program.cs

+5-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ public static async Task<int> Main(string[] args)
1919
rootCommand.AddCommand(new RunCommand());
2020
rootCommand.AddCommand(new ListCommand());
2121
rootCommand.AddCommand(new InitCommand());
22+
rootCommand.AddCommand(new SettingsCommand());
2223

2324
// execute Run if no command provided
2425
RunCommand.AddRunOptions(rootCommand);
@@ -66,10 +67,13 @@ public static async Task<int> Main(string[] args)
6667
}).ConfigureServices((hostContext, services) =>
6768
{
6869
services.AddTransient<IExtensionLoader, ExtensionLoader>();
70+
services.AddTransient<IRawOutputWriter, ConsoleOutputWriter>();
71+
services.AddTransient<IExtensionManifestBuilder, ExtensionManifestBuilder>();
6972
})
7073
.UseCommandHandler<RunCommand, RunCommand.CommandHandler>()
7174
.UseCommandHandler<ListCommand, ListCommand.CommandHandler>()
72-
.UseCommandHandler<InitCommand, InitCommand.CommandHandler>();
75+
.UseCommandHandler<InitCommand, InitCommand.CommandHandler>()
76+
.UseCommandHandler<SettingsCommand, SettingsCommand.CommandHandler>();
7377
})
7478
.UseHelp(AddAdditionalArgumentsHelp)
7579
.UseDefaults().Build();

‎Core/Cosmos.DataTransfer.Core/Properties/launchSettings.json

+4
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@
3131
"Parquet->JSON": {
3232
"commandName": "Project",
3333
"commandLineArgs": "run --sink json-file(beta) --source parquet --settings C:\\Temp\\Parquet-Json.json"
34+
},
35+
"SettingsCommand": {
36+
"commandName": "Project",
37+
"commandLineArgs": "settings --sink -e cosmos-nosql"
3438
}
3539
}
3640
}

0 commit comments

Comments
 (0)