Merging dev updates

Author: bowencode

.github/actions/build-with-plugins/action.yml

@@ -10,6 +10,9 @@ inputs:
   runtime:
     description: Target platform runtime (win-x64)
     required: true
+  build-version:
+    description: Build or release version (0.0.1)
+    required: true
 runs:
   using: composite
   steps:
@@ -27,7 +30,7 @@ runs:
         -p:EnableCompressionInSingleFile=true \
         -p:PublishReadyToRun=false \
         -p:PublishTrimmed=false \
-        -p:Version=0.0.${{ github.run_number }}
+        -p:Version=${{ inputs.build-version }}
     - name: Build Cosmos Extension
       shell: bash
       run: |
@@ -42,7 +45,7 @@ runs:
         -p:EnableCompressionInSingleFile=true \
         -p:PublishReadyToRun=false \
         -p:PublishTrimmed=false \
-        -p:Version=0.0.${{ github.run_number }}
+        -p:Version=${{ inputs.build-version }}
     - name: Build JSON Extension
       shell: bash
       run: |
@@ -57,7 +60,7 @@ runs:
         -p:EnableCompressionInSingleFile=true \
         -p:PublishReadyToRun=false \
         -p:PublishTrimmed=false \
-        -p:Version=0.0.${{ github.run_number }}
+        -p:Version=${{ inputs.build-version }}
     - name: Build Azure Table Extension
       shell: bash
       run: |
@@ -72,7 +75,7 @@ runs:
         -p:EnableCompressionInSingleFile=true \
         -p:PublishReadyToRun=false \
         -p:PublishTrimmed=false \
-        -p:Version=0.0.${{ github.run_number }}
+        -p:Version=${{ inputs.build-version }}
     - name: Build Mongo Extension
       shell: bash
       run: |
@@ -87,7 +90,7 @@ runs:
         -p:EnableCompressionInSingleFile=true \
         -p:PublishReadyToRun=false \
         -p:PublishTrimmed=false \
-        -p:Version=0.0.${{ github.run_number }}
+        -p:Version=${{ inputs.build-version }}
     - name: Build SQL Server Extension
       shell: bash
       run: |
@@ -102,7 +105,37 @@ runs:
         -p:EnableCompressionInSingleFile=true \
         -p:PublishReadyToRun=false \
         -p:PublishTrimmed=false \
-        -p:Version=0.0.${{ github.run_number }}
+        -p:Version=${{ inputs.build-version }}
+    - name: Build Parquet Extension
+      shell: bash
+      run: |
+        dotnet publish \
+        Extensions/Parquet/Cosmos.DataTransfer.ParquetExtension/Cosmos.DataTransfer.ParquetExtension.csproj \
+        --configuration Release \
+        --output ${{ inputs.platform-short }}/Extensions \
+        --self-contained false \
+        --runtime ${{ inputs.runtime }} \
+        -p:PublishSingleFile=false \
+        -p:DebugType=embedded \
+        -p:EnableCompressionInSingleFile=true \
+        -p:PublishReadyToRun=false \
+        -p:PublishTrimmed=false \
+        -p:Version=${{ inputs.build-version }}
+    - name: Build Cognitive Search Extension
+      shell: bash
+      run: |
+        dotnet publish \
+        Extensions/CognitiveSearch/Cosmos.DataTransfer.CognitiveSearchExtension/Cosmos.DataTransfer.CognitiveSearchExtension.csproj \
+        --configuration Release \
+        --output ${{ inputs.platform-short }}/Extensions \
+        --self-contained false \
+        --runtime ${{ inputs.runtime }} \
+        -p:PublishSingleFile=false \
+        -p:DebugType=embedded \
+        -p:EnableCompressionInSingleFile=true \
+        -p:PublishReadyToRun=false \
+        -p:PublishTrimmed=false \
+        -p:Version=${{ inputs.build-version }}
     - name: Upload package
       uses: actions/upload-artifact@v3
       with:

.github/workflows/dotnet-build-test.yml

@@ -1,4 +1,4 @@
-name: .NET
+name: Build and Test all .NET projects
 
 on:
   push:
@@ -8,9 +8,9 @@ on:
 
 jobs:
   build:
-
+    name: Build and test .NET projects
     runs-on: ubuntu-latest
-
+    container: mcr.microsoft.com/dotnet/sdk:6.0
     steps:
     - uses: actions/checkout@v3
     - name: Setup .NET
@@ -28,4 +28,4 @@ jobs:
       uses: actions/upload-artifact@v3
       with:
         name: debug-build
-        path: /home/runner/work/azure-documentdb-datamigrationtool/azure-documentdb-datamigrationtool/Core/Cosmos.DataTransfer.Core/bin/Debug/net6.0 #path/to/artifact/ # or path/to/artifact  
+        path: /home/runner/work/data-migration-desktop-tool/data-migration-desktop-tool/Core/Cosmos.DataTransfer.Core/bin/Debug/net6.0 #path/to/artifact/ # or path/to/artifact  

.github/workflows/internal-test-build.yml

@@ -22,17 +22,20 @@ jobs:
           platform: windows
           platform-short: win
           runtime: win-x64
+          build-version: 0.0.${{ github.run_number }}
       - name: Execute Action build-with-plugins for MacOS
         uses: ./.github/actions/build-with-plugins
         id: build-with-plugins-mac
         with:
           platform: mac
           platform-short: mac
           runtime: osx-x64
+          build-version: 0.0.${{ github.run_number }}
       - name: Execute Action build-with-plugins for Linux
         uses: ./.github/actions/build-with-plugins
         id: build-with-plugins-linux
         with:
           platform: linux
           platform-short: linux
           runtime: linux-x64
+          build-version: 0.0.${{ github.run_number }}

.github/workflows/release.yml

@@ -26,16 +26,16 @@ jobs:
           path: linux-package
       - name: Package output files
         run: |
-          zip -r dmt-${{ github.ref_name }}-win-x64.zip windows-package/*
-          zip -r dmt-${{ github.ref_name }}-mac-x64.zip mac-package/*
-          zip -r dmt-${{ github.ref_name }}-linux-x64.zip linux-package/*
+          zip -r dmt-${{ github.event.release.tag_name }}-win-x64.zip windows-package/*
+          zip -r dmt-${{ github.event.release.tag_name }}-mac-x64.zip mac-package/*
+          zip -r dmt-${{ github.event.release.tag_name }}-linux-x64.zip linux-package/*
       - name: Create GitHub release
         uses: softprops/action-gh-release@v1
         with:
           files: |
-            dmt-${{ github.ref_name }}-win-x64.zip
-            dmt-${{ github.ref_name }}-mac-x64.zip
-            dmt-${{ github.ref_name }}-linux-x64.zip
+            dmt-${{ github.event.release.tag_name }}-win-x64.zip
+            dmt-${{ github.event.release.tag_name }}-mac-x64.zip
+            dmt-${{ github.event.release.tag_name }}-linux-x64.zip
   build-package:
     name: Build self-contained executables
     runs-on: ubuntu-latest
@@ -52,17 +52,20 @@ jobs:
           platform: windows
           platform-short: win
           runtime: win-x64
+          build-version: ${{ github.event.release.tag_name }}
       - name: Execute Action build-with-plugins for MacOS
         uses: ./.github/actions/build-with-plugins
         id: build-with-plugins-mac
         with:
           platform: mac
           platform-short: mac
           runtime: osx-x64
+          build-version: ${{ github.event.release.tag_name }}
       - name: Execute Action build-with-plugins for Linux
         uses: ./.github/actions/build-with-plugins
         id: build-with-plugins-linux
         with:
           platform: linux
           platform-short: linux
           runtime: linux-x64
+          build-version: ${{ github.event.release.tag_name }}

.github/workflows/validate.yml

@@ -1,8 +1,9 @@
 name: Validate all .NET projects
 on:
-  pull_request:
-    branches:
-      - main
+  # pull_request:
+  #   branches:
+  #     - main
+  workflow_dispatch:
 jobs:
   build-test:
     name: Build and test .NET projects

.gitignore

@@ -433,3 +433,4 @@ FodyWeavers.xsd
 
 # macOS .DS_Store
 **/.DS_Store
+/Core/Cosmos.DataTransfer.Core/migrationsettings.json

Core/Cosmos.DataTransfer.Core.UnitTests/SettingsCommandTests.cs

@@ -0,0 +1,92 @@
+using Cosmos.DataTransfer.Interfaces;
+using Cosmos.DataTransfer.Interfaces.Manifest;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using System;
+using System.Collections.Generic;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.ComponentModel.DataAnnotations;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Cosmos.DataTransfer.Core.UnitTests
+{
+    [TestClass]
+    public class SettingsCommandTests
+    {
+        [TestMethod]
+        public void Invoke_ForTestExtension_ProducesValidSettingsJson()
+        {
+            var command = new SettingsCommand();
+            const string source = "testSource";
+            var loader = new Mock<IExtensionManifestBuilder>();
+            var sourceExtension = new Mock<IDataSourceExtensionWithSettings>();
+            sourceExtension.SetupGet(ds => ds.DisplayName).Returns(source);
+            sourceExtension.Setup(ds => ds.GetSettings()).Returns(new List<IDataExtensionSettings>
+            {
+                new MockExtensionSettings(),
+                new MockExtensionSettings2()
+            });
+            loader
+                .Setup(l => l.GetSources())
+                .Returns(new List<IDataSourceExtension> { sourceExtension.Object });
+
+            var writer = new Mock<IRawOutputWriter>();
+            var outputLines = new List<string>();
+            writer.Setup(w => w.WriteLine(It.IsAny<string>())).Callback<string>(s => outputLines.Add(s));
+            var handler = new SettingsCommand.CommandHandler(loader.Object, writer.Object, NullLogger<SettingsCommand.CommandHandler>.Instance)
+            {
+                Source = true,
+                Extension = source
+            };
+
+            var parseResult = new SettingsCommand().Parse(Array.Empty<string>());
+            var result = handler.Invoke(new InvocationContext(parseResult));
+            Assert.AreEqual(0, result);
+
+            bool jsonStarted = false;
+            var stringBuilder = new StringBuilder();
+            foreach (string item in outputLines)
+            {
+                if (item == "<<<")
+                    jsonStarted = true;
+                else if (item == ">>>")
+                    jsonStarted = false;
+                else if (jsonStarted)
+                    stringBuilder.AppendLine(item);
+            }
+
+            var options = new JsonSerializerOptions
+            {
+                Converters = { new JsonStringEnumConverter() },
+                WriteIndented = true
+            };
+            var fullJson = stringBuilder.ToString().Trim();
+            var parsed = JsonSerializer.Deserialize<List<ExtensionSettingProperty>>(fullJson, options);
+            var parsedJson = JsonSerializer.Serialize<List<ExtensionSettingProperty>>(parsed, options);
+
+            Assert.AreEqual(fullJson, parsedJson);
+        }
+    }
+
+    public class MockExtensionSettings : IDataExtensionSettings
+    {
+        [Required]
+        public string Name { get; set; } = null!;
+        public string? Description { get; set; }
+        public int Count { get; set; } = 99;
+        public int? MaxValue { get; set; }
+        public double? Avg { get; set; }
+        public bool Enabled { get; set; } = true;
+        public JsonCommentHandling TestEnum { get; set; }
+    }
+
+    public class MockExtensionSettings2 : IDataExtensionSettings
+    {
+        [Required]
+        public string? AnotherProperty { get; set; }
+    }
+}

Core/Cosmos.DataTransfer.Core/Cosmos.DataTransfer.Core.csproj

@@ -19,7 +19,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Core" Version="1.25.0" />
+    <PackageReference Include="Azure.Core" Version="1.31.0" />
     <PackageReference Include="Microsoft.Data.SqlClient" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />

Core/Cosmos.DataTransfer.Core/DataTransferOptions.cs

@@ -3,6 +3,9 @@
 public class DataTransferOptions
 {
     public string? Source { get; set; }
-    public string? Sink { get; set; }
+    private string? _sink;
+    public string? Sink { get => _sink; set => _sink = value; }
+    public string? Target { get => _sink; set => _sink = value; }
+    public string? Destination { get => _sink; set => _sink = value; }
     public string? SettingsPath { get; set; }
 }