Runs all gfx unit tests through a 'test proxy' (shader-slang#1981)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Support for test proxy.

* Turn on testing using proxy.

* Don't pass sink into check of downstream compiler.

* Small change to kick off build.

* Remove register specification on transcendental.

* Increase poll timeout.
Small improvements to proxy.

* Disable gfx unit tests.

* Put test runner in shared library mode by default.

* Change comment. Kick off another CI test.

* Small edit to kick off builds.

* Run unit tests on proxy.

* Turn on using proxy for now.

* Enable swift shader.

* Fix typo.
Add exception support.

* Make the default spwan type SharedLibrary
Use isolation for gfx unit tests.

* Update slang-binaries.

* Fix typo.

* Report unit test output information.

Author: jsmall-zzz

premake5.lua

@@ -756,6 +756,13 @@ end
 
      links { "compiler-core", "core" }
 
+ 
+ tool "test-proxy"
+     uuid "BE412850-4BB9-429A-877C-BFBC4B34186C"
+     includedirs { "." }
+ 
+     links { "compiler-core", "core", "slang" }
+ 
  --
  -- `slang-generate` is a tool we use for source code generation on
  -- the compiler. It depends on the `core` library, so we need to

slang.sln

@@ -146,9 +146,9 @@ namespace Slang {
                 return SLANG_FAIL;
             }
 
-            // Set a timeout of twenty seconds;
+            // Set a timeout of 100 seconds;
             // we really shouldn't wait too long...
-            int pollTimeout = 20000;
+            int pollTimeout = 100000;
             int pollResult = poll(pollInfos, pollInfoCount, pollTimeout);
             if (pollResult <= 0)
             {

source/core/unix/slang-unix-process-util.cpp

@@ -93,10 +93,10 @@ namespace Slang
         if (type == PassThroughMode::GenericCCpp)
         {
             // try testing for availability on all C/C++ compilers
-            getOrLoadDownstreamCompiler(PassThroughMode::Clang, sink);
-            getOrLoadDownstreamCompiler(PassThroughMode::Gcc, sink);
-            getOrLoadDownstreamCompiler(PassThroughMode::VisualStudio, sink);
-            getOrLoadDownstreamCompiler(PassThroughMode::LLVM, sink);
+            getOrLoadDownstreamCompiler(PassThroughMode::Clang, nullptr);
+            getOrLoadDownstreamCompiler(PassThroughMode::Gcc, nullptr);
+            getOrLoadDownstreamCompiler(PassThroughMode::VisualStudio, nullptr);
+            getOrLoadDownstreamCompiler(PassThroughMode::LLVM, nullptr);
         }
 
         // Mark that we have tried to load it

source/slang/slang-check.cpp

@@ -8,7 +8,7 @@
 //DISABLE_TEST(compute,vulkan):COMPARE_COMPUTE:-vk -output-using-type -shaderobj
 
 //TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0], stride=4):out,name=outputBuffer
-RWStructuredBuffer<float> outputBuffer : register(u0);
+RWStructuredBuffer<float> outputBuffer;
 
 float quantize(double value)
 {

tests/compute/transcendental-double.slang

@@ -4,7 +4,7 @@
 //TEST(compute,vulkan):COMPARE_COMPUTE:-vk -output-using-type -shaderobj
 
 //TEST_INPUT:ubuffer(data=[0 0 0 0 0 0 0 0], stride=4):out,name=outputBuffer
-RWStructuredBuffer<float> outputBuffer : register(u0);
+RWStructuredBuffer<float> outputBuffer;
 
 float quantize(float value)
 {

tests/compute/transcendental.slang

@@ -125,9 +125,9 @@ static bool _isSubCommand(const char* arg)
             }
             optionsOut->binDir = *argCursor++;
         }
-        else if (strcmp(arg, "-useexes") == 0)
+        else if (strcmp(arg, "-use-test-proxy") == 0)
         {
-            optionsOut->useExes = true;
+            optionsOut->defaultSpawnType = SpawnType::UseProxy;
         }
         else if (strcmp(arg, "-v") == 0)
         {

tools/slang-test/options.cpp

@@ -36,6 +36,13 @@ struct TestCategorySet
     Slang::Dictionary<Slang::String, Slang::RefPtr<TestCategory> > m_categoryMap;
 };
 
+enum class SpawnType
+{
+    UseExe,
+    UseSharedLibrary,
+    UseProxy,
+};
+
 struct Options
 {
     char const* appName = "slang-test";
@@ -69,9 +76,13 @@ struct Options
     // integration builds.
     bool dumpOutputOnFailure = false;
 
-    // If set, will force using of executables (not shared library) for tests
-    bool useExes = false;
-
+    // Set the default spawn type to use
+    // Set to SpawnType::UseProxy, if isolation of test execution is desired.
+    // Having tests isolated, slows down testing considerably, so using UseSharedLibrary is the most
+    // desirable default usually.
+    SpawnType defaultSpawnType = SpawnType::UseSharedLibrary;
+    //SpawnType defaultSpawnType = SpawnType::UseProxy;
+    
     // kind of output to generate
     TestOutputMode outputMode = TestOutputMode::Default;
 

tools/slang-test/options.h

@@ -97,11 +97,6 @@ struct FileTestList
     List<TestDetails> tests;
 };
 
-enum class SpawnType
-{
-    UseExe,
-    UseSharedLibrary,
-};
 
 struct TestInput
 {
@@ -531,6 +526,8 @@ static SlangResult _gatherTestsForFile(
     return SLANG_OK;
 }
 
+
+
 Result spawnAndWaitExe(TestContext* context, const String& testPath, const CommandLine& cmdLine, ExecuteResult& outRes)
 {
     const auto& options = context->options;
@@ -550,6 +547,7 @@ Result spawnAndWaitExe(TestContext* context, const String& testPath, const Comma
     return res;
 }
 
+
 Result spawnAndWaitSharedLibrary(TestContext* context, const String& testPath, const CommandLine& cmdLine, ExecuteResult& outRes)
 {
     const auto& options = context->options;
@@ -620,6 +618,44 @@ Result spawnAndWaitSharedLibrary(TestContext* context, const String& testPath, c
 }
 
 
+Result spawnAndWaitProxy(TestContext* context, const String& testPath, const CommandLine& inCmdLine, ExecuteResult& outRes)
+{
+    // Get the name of the thing to execute
+    String exeName = Path::getFileNameWithoutExt(inCmdLine.m_executable);
+
+    if (exeName == "slangc")
+    {
+        // If the test is slangc there is a command line version we can just directly use
+        //return spawnAndWaitExe(context, testPath, inCmdLine, outRes);
+        return spawnAndWaitSharedLibrary(context, testPath, inCmdLine, outRes);
+    }
+
+    CommandLine cmdLine(inCmdLine);
+
+    // Make the first arg the name of the tool to invoke
+    cmdLine.m_args.insert(0, exeName);
+
+    auto exePath = Path::combine(Path::getParentDirectory(inCmdLine.m_executable), String("test-proxy") + ProcessUtil::getExecutableSuffix());
+    cmdLine.setExecutablePath(exePath);
+
+    const auto& options = context->options;
+    if (options.shouldBeVerbose)
+    {
+        String commandLine = ProcessUtil::getCommandLineString(cmdLine);
+        context->reporter->messageFormat(TestMessageType::Info, "%s\n", commandLine.begin());
+    }
+
+    // Execute
+    Result res = ProcessUtil::execute(cmdLine, outRes);
+    if (SLANG_FAILED(res))
+    {
+        //        fprintf(stderr, "failed to run test '%S'\n", testPath.ToWString());
+        context->reporter->messageFormat(TestMessageType::RunError, "failed to run test '%S'", testPath.toWString().begin());
+    }
+
+    return res;
+}
+
 static SlangResult _extractArg(const CommandLine& cmdLine, const String& argName, String& outValue)
 {
     SLANG_ASSERT(argName.getLength() > 0 && argName[0] == '-');
@@ -972,6 +1008,11 @@ ToolReturnCode spawnAndWait(TestContext* context, const String& testPath, SpawnT
             spawnResult = spawnAndWaitSharedLibrary(context, testPath, cmdLine, outExeRes);
             break;
         }
+        case SpawnType::UseProxy:
+        {
+            spawnResult = spawnAndWaitProxy(context, testPath, cmdLine, outExeRes);
+            break;
+        }
         default: break;
     }
 
@@ -2880,8 +2921,6 @@ TestResult runTest(
         return TestResult::Pass;
     }
 
-    const SpawnType defaultSpawnType = context->options.useExes ? SpawnType::UseExe : SpawnType::UseSharedLibrary;
-
     auto testInfo = _findTestCommandInfoByCommand(testOptions.command.getUnownedSlice());
 
     if (testInfo)
@@ -2890,7 +2929,7 @@ TestResult runTest(
         testInput.filePath = filePath;
         testInput.outputStem = outputStem;
         testInput.testOptions = &testOptions;
-        testInput.spawnType = defaultSpawnType;
+        testInput.spawnType = context->options.defaultSpawnType;
 
         return testInfo->callback(context, testInput);
     }
@@ -3346,8 +3385,18 @@ static void _disableCPPBackends(TestContext* context)
     }
 }
 
+static TestResult _asTestResult(ToolReturnCode retCode)
+{
+    switch (retCode)
+    {
+        default:                        return TestResult::Fail;
+        case ToolReturnCode::Success:   return TestResult::Pass;
+        case ToolReturnCode::Ignored:   return TestResult::Ignored;
+    }
+}
+
     /// Loads a DLL containing unit test functions and run them one by one.
-static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOptions, const char* moduleName)
+static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOptions, SpawnType spawnType, const char* moduleName)
 {
     SharedLibrary::Handle moduleHandle;
     SLANG_RETURN_ON_FAIL(SharedLibrary::load(
@@ -3368,6 +3417,9 @@ static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOpti
     unitTestContext.workDirectory = "";
     unitTestContext.enabledApis = context->options.enabledApis;
     auto testCount = testModule->getTestCount();
+
+    TestReporter* reporter = TestReporter::get();
+
     for (SlangInt i = 0; i < testCount; i++)
     {
         auto testFunc = testModule->getTestFunc(i);
@@ -3382,9 +3434,50 @@ static SlangResult runUnitTestModule(TestContext* context, TestOptions& testOpti
         {
             if (testPassesCategoryMask(context, testOptions))
             {
-                TestReporter::get()->startTest(testOptions.command.getBuffer());
-                testFunc(&unitTestContext);
-                TestReporter::get()->endTest();
+                if (spawnType == SpawnType::UseProxy)
+                {
+                    CommandLine cmdLine;
+
+                    // The 'command' is the module 
+                    cmdLine.setExecutablePath(Path::combine(context->exeDirectoryPath, moduleName));
+
+                    // Pass the test name / index
+                    cmdLine.addArg(testName);
+
+                    {
+                        StringBuilder buf;
+                        buf << i;
+                        cmdLine.addArg(buf.ProduceString());
+                    }
+
+                    // Pass the enabled apis
+                    {
+                        StringBuilder buf;
+                        buf << context->options.enabledApis;
+                        cmdLine.addArg(buf.ProduceString());
+                    }
+
+                    {
+                        TestReporter::TestScope scopeTest(reporter, testOptions.command);
+                        ExecuteResult exeRes;
+
+                        const auto testResult = _asTestResult(spawnAndWait(context, filePath, spawnType, cmdLine, exeRes));
+
+                        // If the test fails, output any output - which might give information about individual tests that have failed.
+                        if (testResult == TestResult::Fail)
+                        {
+                            String output = getOutput(exeRes);
+                            reporter->message(TestMessageType::TestFailure, output.getBuffer());
+                        }
+
+                        reporter->addResult(testResult);
+                    }
+                }
+                else
+                {
+                    TestReporter::TestScope scopeTest(reporter, testOptions.command);
+                    testFunc(&unitTestContext);
+                }
             }
         }
     }
@@ -3570,13 +3663,15 @@ SlangResult innerMain(int argc, char** argv)
                 TestOptions testOptions;
                 testOptions.categories.add(unitTestCategory);
                 testOptions.categories.add(smokeTestCategory);
-                runUnitTestModule(&context, testOptions, "slang-unit-test-tool");
+                runUnitTestModule(&context, testOptions, context.options.defaultSpawnType, "slang-unit-test-tool");
             }
+            
             {
                 TestOptions testOptions;
                 testOptions.categories.add(unitTestCategory);
-                runUnitTestModule(&context, testOptions, "gfx-unit-test-tool");
+                runUnitTestModule(&context, testOptions, SpawnType::UseProxy, "gfx-unit-test-tool");
             }
+
             TestReporter::set(nullptr);
         }