Emit errors when seeing a matrix with row/col count of 1

Such matrices aren't well supported except by D3D targets.
Therefore, generate an error rather than outputting invalid code for non-D3D targets.

This closes shader-slang#5987.

Author: aleino-nv

source/slang/slang-diagnostic-defs.h

@@ -1658,6 +1658,12 @@ DIAGNOSTIC(
     overloadedParameterToHigherOrderFunction,
     "passing overloaded functions to higher order functions is not supported")
 
+DIAGNOSTIC(
+    39999,
+    Error,
+    matrixColumnOrRowCountIsOne,
+    "matrices with 1 column or row are not supported by the current code generation target")
+
 // 38xxx
 
 DIAGNOSTIC(

source/slang/slang-emit.cpp

@@ -555,6 +555,24 @@ static void unexportNonEmbeddableIR(CodeGenTarget target, IRModule* irModule)
     }
 }
 
+static void validateMatrixDimensions(DiagnosticSink* sink, IRModule* module)
+{
+    for (auto globalInst : module->getGlobalInsts())
+    {
+        if (auto matrixType = as<IRMatrixType>(globalInst))
+        {
+            auto colCount = as<IRIntLit>(matrixType->getColumnCount());
+            auto rowCount = as<IRIntLit>(matrixType->getRowCount());
+
+            if ((rowCount && (rowCount->getValue() == 1)) ||
+                (colCount && (colCount->getValue() == 1)))
+            {
+                sink->diagnose(matrixType->sourceLoc, Diagnostics::matrixColumnOrRowCountIsOne);
+            }
+        }
+    }
+}
+
 Result linkAndOptimizeIR(
     CodeGenContext* codeGenContext,
     LinkingAndOptimizationOptions const& options,
@@ -1504,6 +1522,10 @@ Result linkAndOptimizeIR(
 #endif
     validateIRModuleIfEnabled(codeGenContext, irModule);
 
+    // Make sure there are no matrices with 1 row/column, except for D3D targets where it's allowed.
+    if (!isD3DTarget(targetRequest))
+        validateMatrixDimensions(sink, irModule);
+
     // The resource-based specialization pass above
     // may create specialized versions of functions, but
     // it does not try to completely eliminate the original