Add SQL file for SqlServer

.github/workflows/dotnet-build-test.yml

@@ -29,3 +29,15 @@ jobs:
       with:
         name: debug-build
         path: /home/runner/work/data-migration-desktop-tool/data-migration-desktop-tool/Core/Cosmos.DataTransfer.Core/bin/Debug/net8.0 #path/to/artifact/ # or path/to/artifact  
+    - name: Code Coverage Report
+      uses: irongut/CodeCoverageSummary@v1.3.0
+      with:
+        filename: ./**/coverage.cobertura.xml
+        badge: true
+        fail_below_min: false
+        format: markdown
+        hide_branch_rate: false
+        hide_complexity: true
+        indicators: true
+        output: both
+        thresholds: '60 80'

.github/workflows/validate.yml

@@ -18,3 +18,15 @@ jobs:
         run: dotnet build
       - name: Run unit tests
         run: dotnet test --logger "console;verbosity=detailed"
+      - name: Code Coverage Report
+        uses: irongut/CodeCoverageSummary@v1.3.0
+        with:
+          filename: ./**/coverage.cobertura.xml
+          badge: true
+          fail_below_min: false
+          format: markdown
+          hide_branch_rate: false
+          hide_complexity: true
+          indicators: true
+          output: both
+          thresholds: '60 80'

.gitignore

@@ -68,6 +68,9 @@ bld/
 # Visual Studio 2017 auto generated files
 Generated\ Files/
 
+# MSTest .runsettings
+*.runsettings
+
 # MSTest test Results
 [Tt]est[Rr]esult*/
 [Bb]uild[Ll]og.*
@@ -434,3 +437,4 @@ FodyWeavers.xsd
 # macOS .DS_Store
 **/.DS_Store
 /Core/Cosmos.DataTransfer.Core/migrationsettings.json
+

Directory.Packages.props

@@ -17,6 +17,7 @@
     <PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.46.1" />
     <PackageVersion Include="Microsoft.Azure.Cosmos.Encryption" Version="2.0.4" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="6.0.1" />
+    <PackageVersion Include="Microsoft.Data.Sqlite" Version="9.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />

Extensions/Csv/Cosmos.DataTransfer.CsvExtension.UnitTests/Cosmos.DataTransfer.CsvExtension.UnitTests.csproj

@@ -12,17 +12,17 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="MSTest.TestAdapter" />
     <PackageReference Include="MSTest.TestFramework" />
-    <PackageReference Include="coverlet.collector">
+    <PackageReference Include="Moq" />
+    <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="coverlet.msbuild">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Newtonsoft.Json" />
-    <PackageReference Include="System.Text.Json" />
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\Json\Cosmos.DataTransfer.JsonExtension.UnitTests\Cosmos.DataTransfer.JsonExtension.UnitTests.csproj" />
     <ProjectReference Include="..\Cosmos.DataTransfer.CsvExtension\Cosmos.DataTransfer.CsvExtension.csproj" />
+    <ProjectReference Include="..\..\..\Interfaces\Cosmos.DataTransfer.Common.UnitTests\Cosmos.DataTransfer.Common.UnitTests.csproj" />
   </ItemGroup>
 
   <ItemGroup>

Extensions/Csv/Cosmos.DataTransfer.CsvExtension.UnitTests/CsvFileRoundTripTests.cs

@@ -1,8 +1,6 @@
 using System.Globalization;
-using Cosmos.DataTransfer.JsonExtension;
-using Cosmos.DataTransfer.JsonExtension.UnitTests;
+using Cosmos.DataTransfer.Common.UnitTests;
 using Microsoft.Extensions.Logging.Abstractions;
-using Newtonsoft.Json.Linq;
 
 namespace Cosmos.DataTransfer.CsvExtension.UnitTests;
 

Extensions/Csv/Cosmos.DataTransfer.CsvExtension.UnitTests/CsvFileSourceTests.cs

@@ -1,4 +1,4 @@
-using Cosmos.DataTransfer.JsonExtension.UnitTests;
+using Cosmos.DataTransfer.Common.UnitTests;
 using Microsoft.Extensions.Logging.Abstractions;
 
 namespace Cosmos.DataTransfer.CsvExtension.UnitTests;
@@ -24,7 +24,7 @@ public async Task ReadAsync_WithSimpleFile_ReadsRows()
             object? value = dataItem.GetValue("id");
             Assert.IsNotNull(value);
             Assert.IsNotNull(dataItem.GetValue("name"));
-            var current = Int32.Parse(value.ToString());
+            var current = Int32.Parse(value.ToString()!);
             Assert.IsTrue(current > lastId);
             lastId = current;
         }

Extensions/Csv/Cosmos.DataTransfer.CsvExtension.UnitTests/CsvWriterSettingsTests.cs

@@ -1,10 +1,10 @@
 using System.Globalization;
-using Cosmos.DataTransfer.JsonExtension.UnitTests;
 using Microsoft.Extensions.Logging.Abstractions;
 using System.ComponentModel.DataAnnotations;
 using Cosmos.DataTransfer.CsvExtension.Settings;
 using Cosmos.DataTransfer.Interfaces;
-using Cosmos.DataTransfer.JsonExtension;
+using Cosmos.DataTransfer.Common.UnitTests;
+using Moq;
 
 namespace Cosmos.DataTransfer.CsvExtension.UnitTests;
 
@@ -113,8 +113,9 @@ public async Task TestDanishCulture() {
         };
 
         var sink = new CsvFileSink();
+        var mockDataSource = new Mock<IDataSourceExtension>();
 
-        await sink.WriteAsync(data.ToAsyncEnumerable(), config, new JsonFileSource(), NullLogger.Instance);
+        await sink.WriteAsync(data.ToAsyncEnumerable(), config, mockDataSource.Object, NullLogger.Instance);
         var result = await File.ReadAllTextAsync(outputFile);
         Assert.AreEqual("1,2", result);
     }

Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/Cosmos.DataTransfer.JsonExtension.UnitTests.csproj

@@ -13,17 +13,18 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="MSTest.TestAdapter" />
     <PackageReference Include="MSTest.TestFramework" />
-    <PackageReference Include="coverlet.collector">
+    <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="coverlet.msbuild">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="System.Linq.Async" />
-    <PackageReference Include="System.Text.Json" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Cosmos.DataTransfer.JsonExtension\Cosmos.DataTransfer.JsonExtension.csproj" />
+    <ProjectReference Include="..\..\..\Interfaces\Cosmos.DataTransfer.Common.UnitTests\Cosmos.DataTransfer.Common.UnitTests.csproj" />
   </ItemGroup>
 
   <ItemGroup>

Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonFileRoundTripTests.cs

@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging.Abstractions;
 using Newtonsoft.Json.Linq;
+using Cosmos.DataTransfer.Common.UnitTests;
 
 namespace Cosmos.DataTransfer.JsonExtension.UnitTests
 {

Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonFileSinkTests.cs

@@ -1,4 +1,5 @@
 using Cosmos.DataTransfer.Interfaces;
+using Cosmos.DataTransfer.Common.UnitTests;
 using Microsoft.Extensions.Logging.Abstractions;
 using Newtonsoft.Json;
 

Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonSinkTests.cs

@@ -1,4 +1,5 @@
 using Cosmos.DataTransfer.Interfaces;
+using Cosmos.DataTransfer.Common.UnitTests;
 using Microsoft.Extensions.Logging.Abstractions;
 using Newtonsoft.Json;
 

Extensions/Json/Cosmos.DataTransfer.JsonExtension.UnitTests/JsonSourceTests.cs

@@ -1,4 +1,5 @@
 using Cosmos.DataTransfer.Interfaces;
+using Cosmos.DataTransfer.Common.UnitTests;
 using Microsoft.Extensions.Logging.Abstractions;
 
 namespace Cosmos.DataTransfer.JsonExtension.UnitTests

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension.UnitTests/Cosmos.DataTransfer.SqlServerExtension.UnitTests.csproj

@@ -6,28 +6,34 @@
     <Nullable>enable</Nullable>
 
     <IsPackable>false</IsPackable>
+    <CollectCoverage>true</CollectCoverage>
+    <CoverletOutputFormat>cobertura</CoverletOutputFormat>
+    <CoverletOutput>../../../Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension/coverage.cobertura.xml</CoverletOutput>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Data.Sqlite" />
     <PackageReference Include="Microsoft.Extensions.Configuration" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="MSTest.TestAdapter" />
     <PackageReference Include="MSTest.TestFramework" />
-    <PackageReference Include="coverlet.collector">
+    <PackageReference Include="System.Linq.Async" />
+    <PackageReference Include="Moq" />
+    <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="coverlet.msbuild">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Newtonsoft.Json" />
-    <PackageReference Include="System.Linq.Async" />
-    <PackageReference Include="System.Text.Json" />
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\..\Interfaces\Cosmos.DataTransfer.Common.UnitTests\Cosmos.DataTransfer.Common.UnitTests.csproj" />
     <ProjectReference Include="..\Cosmos.DataTransfer.SqlServerExtension\Cosmos.DataTransfer.SqlServerExtension.csproj" />
   </ItemGroup>
 
+
   <ItemGroup>
-    <None Update="Data\FlatFile.json">
+    <None Update="Data\SqlStatement.sql">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension.UnitTests/Data/SqlStatement.sql

@@ -0,0 +1 @@
+SELECT * FROM foobar;

Extensions/SqlServer/Cosmos.DataTransfer.SqlServerExtension.UnitTests/SqlServerDataSourceExtensionTests.cs

@@ -0,0 +1,115 @@
+using Microsoft.Data.Sqlite;
+using Microsoft.Extensions.Logging.Abstractions;
+using System.Data.Common;
+using Cosmos.DataTransfer.Interfaces;
+using Cosmos.DataTransfer.Common;
+using Cosmos.DataTransfer.Common.UnitTests;
+using Moq;
+using Microsoft.Extensions.Configuration;
+
+namespace Cosmos.DataTransfer.SqlServerExtension.UnitTests;
+
+[TestClass]
+public class SqlServerDataSourceExtensionTests
+{
+
+    private static async Task<Tuple<SqliteFactory,DbConnection>> connectionFactory(CancellationToken cancellationToken = default(CancellationToken)) {
+        var provider = SqliteFactory.Instance;
+        var connection = provider.CreateConnection();
+        await connection.OpenAsync(cancellationToken);
+
+        var cmd = connection.CreateCommand();
+        cmd.CommandText = @"CREATE TABLE foobar (
+            id INTEGER NOT NULL,
+            name TEXT
+        );";
+        await cmd.ExecuteNonQueryAsync(cancellationToken);
+        cmd.CommandText = @"INSERT INTO foobar (id, name) 
+        VALUES (1, 'zoo');";
+        await cmd.ExecuteNonQueryAsync(cancellationToken);
+        cmd.CommandText = @"INSERT INTO foobar (id, name) 
+        VALUES (2, NULL);";
+        await cmd.ExecuteNonQueryAsync(cancellationToken);
+
+        return Tuple.Create(provider, connection);
+    }
+
+    [TestMethod]
+    public async Task TestReadAsync() {
+        var config = new Mock<IConfiguration>();
+        var cancellationToken = new CancellationTokenSource(500);
+        var (providerFactory, connection) = await connectionFactory(cancellationToken.Token);
+
+        var extension = new SqlServerDataSourceExtension();
+        Assert.AreEqual("SqlServer", extension.DisplayName);
+
+        var result = await extension.ReadAsync(config.Object, NullLogger.Instance,
+          "SELECT * FROM foobar", Array.Empty<DbParameter>(), connection, providerFactory, cancellationToken.Token).ToListAsync();
+        var expected = new List<DictionaryDataItem> {
+            new DictionaryDataItem(new Dictionary<string, object?> { { "id", (long)1 }, { "name", "zoo" } }),
+            new DictionaryDataItem(new Dictionary<string, object?> { { "id", (long)2 }, { "name", null }  })
+        };
+        CollectionAssert.That.AreEqual(expected, result, new DataItemComparer());
+    }
+
+    [TestMethod]
+    public async Task TestReadAsyncWithParameters() {
+        var config = new Mock<IConfiguration>();
+        var cancellationToken = new CancellationTokenSource();
+        var (providerFactory, connection) = await connectionFactory(cancellationToken.Token);
+
+        var extension = new SqlServerDataSourceExtension();
+        Assert.AreEqual("SqlServer", extension.DisplayName);
+
+        var parameter = providerFactory.CreateParameter();
+        parameter.ParameterName = "@x";
+        parameter.DbType = System.Data.DbType.Int32;
+        parameter.Value = 2;
+
+        var result = await extension.ReadAsync(config.Object, NullLogger.Instance,
+          "SELECT * FROM foobar WHERE id = @x", 
+          new DbParameter[] { parameter }, connection, providerFactory, cancellationToken.Token).FirstAsync();
+        Assert.That.AreEqual(result,
+            new DictionaryDataItem(new Dictionary<string, object?> { { "id", (long)2 }, { "name", null } }),
+            new DataItemComparer());
+    }
+
+    // Allows for testing against an actual SQL Server by specifying a 
+    // connectionstring in either a .runsettings-file or environment variable.
+    // Example: Using sql.runsettings
+    // <?xml version="1.0" encoding="utf-8"?>
+    // <RunSettings>
+    //   <TestRunParameters>
+    //     <Parameter name="TestReadAsync_LiveSqlServer_ConnectionString" value="Your connection string" />
+    //   </TestRunParameters>
+    // </RunSettings>
+    // run test with
+    // dotnet test --settings sql.runsettings 
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+    public TestContext TestContext { get; set; }
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+    [TestMethod]
+    [Timeout(1000)]
+    public async Task TestReadAsync_LiveSqlServer() {
+        var connectionString = (string?)TestContext.Properties["TestReadAsync_LiveSqlServer_ConnectionString"];
+        connectionString ??= Environment.GetEnvironmentVariable("TestReadAsync_LiveSqlServer_ConnectionString");
+        if (connectionString is null) {
+            Assert.Inconclusive("Could not run, as no connection string to live SQL Server was provided.");
+        }
+
+        var extension = new SqlServerDataSourceExtension();
+        var config = TestHelpers.CreateConfig(new Dictionary<string, string> {
+            { "ConnectionString", connectionString! },
+            { "QueryText", "SELECT 1, 'foo' as bar, NULL as zoo;" }
+        });
+
+        var result = await extension.ReadAsync(config, NullLogger.Instance).FirstAsync();
+
+        Assert.IsTrue(new DataItemComparer().Equals(result, 
+            new DictionaryDataItem(new Dictionary<string, object?> {
+                { "", 1 },
+                { "bar", "foo" },
+                { "zoo", null }
+        })));
+    }
+}