Test Parquet sink

GuillaumeVadeBe · GuillaumeVadeBe · commit caf047ff2ddd · 2024-12-16T15:14:59.000+01:00
diff --git a/Extensions/Parquet/Cosmos.DataTransfer.ParquetExtension/ParquetDataCol.cs b/Extensions/Parquet/Cosmos.DataTransfer.ParquetExtension/ParquetDataCol.cs
@@ -33,6 +33,7 @@ public ParquetDataCol(string name, Type coltype)
         {
             ColumnName = name;
             ColumnType = coltype;
+
             if (coltype != System.Type.Missing.GetType())
             {
                 ParquetDataType = MapDataType(name, coltype);
diff --git a/Extensions/Parquet/Cosmos.DataTransfer.ParquetExtension/ParquetFormatWriter.cs b/Extensions/Parquet/Cosmos.DataTransfer.ParquetExtension/ParquetFormatWriter.cs
@@ -42,6 +42,10 @@ private void ProcessColumns(IDataItem item, long row)
                 {
                     coltype = colval.GetType();
                 }
+                if (col == "Timestamp" && coltype == typeof(DateTimeOffset))
+                {
+                    continue;
+                }
                 if (current == null)
                 {
                     var newcol = new ParquetDataCol(col, coltype);

Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,7 @@ public ParquetDataCol(string name, Type coltype)`
`33`	`33`	`{`
`34`	`34`	`ColumnName = name;`
`35`	`35`	`ColumnType = coltype;`
	`36`	`+`
`36`	`37`	`if (coltype != System.Type.Missing.GetType())`
`37`	`38`	`{`
`38`	`39`	`ParquetDataType = MapDataType(name, coltype);`
Original file line number	Diff line number	Diff line change
`@@ -42,6 +42,10 @@ private void ProcessColumns(IDataItem item, long row)`
`42`	`42`	`{`
`43`	`43`	`coltype = colval.GetType();`
`44`	`44`	`}`
	`45`	`+ if (col == "Timestamp" && coltype == typeof(DateTimeOffset))`
	`46`	`+ {`
	`47`	`+ continue;`
	`48`	`+ }`
`45`	`49`	`if (current == null)`
`46`	`50`	`{`
`47`	`51`	`var newcol = new ParquetDataCol(col, coltype);`