GitHub action benchmark (shader-slang#4804)

Adds a new Github CI action for benchmarking the slangc compiler on the MDL shaders. For now, the results are only dumped to the output of the CI, which can be later viewed through raw logs. The next step is to use github-action-benchmark to push these results into a page which will show the benchmark results over time as commits are pushed.

Author: venkataram-nv

.github/workflows/benchmark.yml

@@ -0,0 +1,47 @@
+name: Benchmark
+
+on:
+  push:
+    branches: [master]
+    paths-ignore:
+      - 'docs/**'
+      - 'LICENCE'
+      - 'CONTRIBUTION.md'
+      - 'README.md'
+  pull_request:
+    branches: [master]
+    paths-ignore:
+      - 'docs/**'
+      - 'LICENCE'
+      - 'CONTRIBUTION.md'
+      - 'README.md'
+
+jobs:
+  build:
+    runs-on: [Windows, benchmark, self-hosted]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+          fetch-depth: '0'
+      - name: Common setup
+        uses: ./.github/actions/common-setup
+        with:
+          os: windows
+          compiler: cl
+          platform: x86_64
+          config: release
+          build-llvm: true
+      - name: Build Slang
+        run: |
+          cmake --preset default --fresh -DSLANG_SLANG_LLVM_FLAVOR=USE_SYSTEM_LLVM -DCMAKE_COMPILE_WARNING_AS_ERROR=false
+          cmake --workflow --preset release
+      - name: Setup
+        run: |
+          cd tests/mdl
+          pip install prettytable argparse
+      - name: Run benchmark
+        run: |
+          cd tests/mdl
+          Copy-Item -Path C:\slang-benchmarks -Destination . -Recurse
+          python compile.py --samples 16 --target dxil --ci

source/core/slang-performance-profiler.cpp

@@ -31,11 +31,14 @@ namespace Slang
         }
         virtual void getResult(StringBuilder& out) override
         {
-            for (auto func : data)
+            char buffer[512];
+            for (const auto& func : data)
             {
-                out << func.key << ": \t";
+                memset(buffer, 0, sizeof(buffer));
+                snprintf(buffer, sizeof(buffer), "[*] %30s", func.key);
+                out << buffer << " \t";
                 auto milliseconds = std::chrono::duration_cast< std::chrono::milliseconds >(func.value.duration);
-                out << func.value.invocationCount << "\t" << milliseconds.count() << "ms\n";
+                out << func.value.invocationCount << " \t" << milliseconds.count() << "ms\n";
             }
         }
         virtual void clear() override

source/core/slang-performance-profiler.h

@@ -68,7 +68,9 @@ struct SlangProfiler: public ISlangProfiler, public RefObject
     List<ProfileInfo> m_profilEntries;
 };
 
-#define SLANG_PROFILE PerformanceProfilerFuncRAIIContext _profileContext(__func__)
+#define SLANG_PROFILE            PerformanceProfilerFuncRAIIContext _profileContext(__func__)
+#define SLANG_PROFILE_SECTION(s) PerformanceProfilerFuncRAIIContext _profileContext##s(#s)
+
 }
 
 #endif

source/slang/slang-compiler.cpp

@@ -3,6 +3,7 @@
 #include "../core/slang-basic.h"
 #include "../core/slang-platform.h"
 #include "../core/slang-io.h"
+#include "../core/slang-performance-profiler.h"
 #include "../core/slang-string-util.h"
 #include "../core/slang-hex-dump-util.h"
 #include "../core/slang-riff.h"
@@ -2297,6 +2298,7 @@ namespace Slang
 
     void EndToEndCompileRequest::generateOutput()
     {
+        SLANG_PROFILE;
         generateOutput(getSpecializedGlobalAndEntryPointsComponentType());
 
         // If we are in command-line mode, we might be expected to actually

source/slang/slang.cpp

@@ -2742,6 +2742,7 @@ static void _outputIncludes(const List<SourceFile*>& sourceFiles, SourceManager*
 void FrontEndCompileRequest::parseTranslationUnit(
     TranslationUnitRequest* translationUnit)
 {
+    SLANG_PROFILE;
     if (translationUnit->isChecked)
         return;
 
@@ -2924,6 +2925,7 @@ void FrontEndCompileRequest::checkAllTranslationUnits()
 
 void FrontEndCompileRequest::generateIR()
 {
+    SLANG_PROFILE;
     SLANG_AST_BUILDER_RAII(getLinkage()->getASTBuilder());
 
     // Our task in this function is to generate IR code
@@ -3021,6 +3023,7 @@ static SourceLanguage inferSourceLanguage(FrontEndCompileRequest* request)
 
 SlangResult FrontEndCompileRequest::executeActionsInner()
 {
+    SLANG_PROFILE_SECTION(frontEndExecute);
     SLANG_AST_BUILDER_RAII(getLinkage()->getASTBuilder());
 
     for (TranslationUnitRequest* translationUnit : translationUnits)
@@ -3046,7 +3049,11 @@ SlangResult FrontEndCompileRequest::executeActionsInner()
         return SLANG_FAIL;
 
     // Perform semantic checking on the whole collection
-    checkAllTranslationUnits();
+    {
+        SLANG_PROFILE_SECTION(SemanticChecking);
+        checkAllTranslationUnits();
+    }
+
     if (getSink()->getErrorCount() != 0)
         return SLANG_FAIL;
 
@@ -3172,6 +3179,7 @@ void EndToEndCompileRequest::init()
 
 SlangResult EndToEndCompileRequest::executeActionsInner()
 {
+    SLANG_PROFILE_SECTION(endToEndActions);
     // If no code-generation target was specified, then try to infer one from the source language,
     // just to make sure we can do something reasonable when invoked from the command line.
     //
@@ -6303,6 +6311,7 @@ SlangResult EndToEndCompileRequest::compile()
     if (getOptionSet().getBoolOption(CompilerOptionName::ReportDownstreamTime))
     {
         getSession()->getCompilerElapsedTime(&totalStartTime, &downstreamStartTime);
+        PerformanceProfiler::getProfiler()->clear();
     }
 #if !defined(SLANG_DEBUG_INTERNAL_ERROR)
     // By default we'd like to catch as many internal errors as possible,
@@ -6317,6 +6326,7 @@ SlangResult EndToEndCompileRequest::compile()
 
     try
     {
+        SLANG_PROFILE_SECTION(compileInner);
         res = executeActions();
     }
     catch (const AbortCompilationException& e)

tests/mdl/.gitignore

@@ -0,0 +1,4 @@
+slang-benchmarks
+targets/
+modules/
+*.json