Slang -> C++ -> SharedLibrary -> Test (shader-slang#999)

* WIP: Adding support for C/C++ compilation to slang API.

* Removed BackEndType in test harness -> use SlangPassThrough to identify backends
Only require stage for targets that require it.
Detection of all different backends.

* Windows/Unix create temporary filename.

* WIP: Output CPU binaries.

* Added a pass-through c/c++ test.

* Compile C++/C and store in temporary file.

* Read the binary back into memory.

* Set debug info and optimization flags for C/C++.
Make the CPPCompiler debug/optimization levels match slangs.

* Handling of include paths and math precision.

* Dumping c++/c source and exe/shared library.

* Put hex dump into own util.

* End to end pass through c compilation test.

* WIP: Simple execute test working on Linux/Unix.

* Fix typo on linux.

* WIP: To compile slang to cpp shared library. Report backend compiler errors.

* Compiles slang -> cpp and loads as shared library.

* Fix problem on c-cross-compile test because prelude is now included with <> quotes.

* Run slang generated cpp code - using hard coded data.

* Added cpp-execute-simple, and test output.

* Fix warning that broke win32 build.

* Fix compilation problem on osx.

Author: jsmall-zzz

slang.h

@@ -514,6 +514,8 @@ extern "C"
         SLANG_DXIL_ASM,
         SLANG_C_SOURCE,             ///< The C language
         SLANG_CPP_SOURCE,           ///< The C++ language
+        SLANG_EXECUTABLE,           ///< Executable (for hosting CPU/OS)
+        SLANG_SHARED_LIBRARY,       ///< A shared library/Dll (for hosting CPU/OS)
     };
 
     /* A "container format" describes the way that the outputs
@@ -537,6 +539,11 @@ extern "C"
         SLANG_PASS_THROUGH_FXC,
         SLANG_PASS_THROUGH_DXC,
         SLANG_PASS_THROUGH_GLSLANG,
+        SLANG_PASS_THROUGH_CLANG,                   ///< Clang C/C++ compiler 
+        SLANG_PASS_THROUGH_VISUAL_STUDIO,           ///< Visual studio C/C++ compiler
+        SLANG_PASS_THROUGH_GCC,                     ///< GCC C/C++ compiler
+        SLANG_PASS_THROUGH_GENERIC_C_CPP,           ///< Generic C or C++ compiler, which is decided by the source type
+        SLANG_PASS_THROUGH_COUNT_OF,
     };
 
     /*!
@@ -601,6 +608,8 @@ extern "C"
         SLANG_SOURCE_LANGUAGE_SLANG,
         SLANG_SOURCE_LANGUAGE_HLSL,
         SLANG_SOURCE_LANGUAGE_GLSL,
+        SLANG_SOURCE_LANGUAGE_C,
+        SLANG_SOURCE_LANGUAGE_CPP,
     };
 
     typedef unsigned int SlangProfileID;
@@ -859,7 +868,7 @@ extern "C"
 
     typedef void(*SlangFuncPtr)(void);
 
-    /** An interface that can be used to encapsulate access to a shared library. An implementaion 
+    /** An interface that can be used to encapsulate access to a shared library. An implementation 
     does not have to implement the library as a shared library. 
     */
     struct ISlangSharedLibrary: public ISlangUnknown

source/core/core.vcxproj

@@ -182,6 +182,7 @@
     <ClInclude Include="slang-free-list.h" />
     <ClInclude Include="slang-gcc-compiler-util.h" />
     <ClInclude Include="slang-hash.h" />
+    <ClInclude Include="slang-hex-dump-util.h" />
     <ClInclude Include="slang-io.h" />
     <ClInclude Include="slang-list.h" />
     <ClInclude Include="slang-math.h" />
@@ -213,6 +214,7 @@
     <ClCompile Include="slang-cpp-compiler.cpp" />
     <ClCompile Include="slang-free-list.cpp" />
     <ClCompile Include="slang-gcc-compiler-util.cpp" />
+    <ClCompile Include="slang-hex-dump-util.cpp" />
     <ClCompile Include="slang-io.cpp" />
     <ClCompile Include="slang-memory-arena.cpp" />
     <ClCompile Include="slang-object-scope-manager.cpp" />

source/core/core.vcxproj.filters

@@ -45,6 +45,9 @@
     <ClInclude Include="slang-hash.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="slang-hex-dump-util.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="slang-io.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -134,6 +137,9 @@
     <ClCompile Include="slang-gcc-compiler-util.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="slang-hex-dump-util.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="slang-io.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>

source/core/slang-cpp-compiler.cpp

@@ -19,6 +19,17 @@
 namespace Slang
 {
 
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompiler::Desc !!!!!!!!!!!!!!!!!!!!!!*/
+
+void CPPCompiler::Desc::appendAsText(StringBuilder& out) const
+{
+    out << getCompilerTypeAsText(type);
+    out << " ";
+    out << majorVersion;
+    out << ".";
+    out << minorVersion;
+}
+
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompiler::OutputMessage !!!!!!!!!!!!!!!!!!!!!!*/
 
 /* static */UnownedStringSlice CPPCompiler::OutputMessage::getTypeText(OutputMessage::Type type)
@@ -33,6 +44,21 @@ namespace Slang
     }
 }
 
+/* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompiler !!!!!!!!!!!!!!!!!!!!!!!!!!!!!*/
+
+/* static */UnownedStringSlice CPPCompiler::getCompilerTypeAsText(CompilerType type)
+{
+    switch (type)
+    {
+        default:
+        case CompilerType::Unknown:     return UnownedStringSlice::fromLiteral("Unknown");
+        case CompilerType::VisualStudio:return UnownedStringSlice::fromLiteral("Visual Studio");
+        case CompilerType::GCC:         return UnownedStringSlice::fromLiteral("GCC");
+        case CompilerType::Clang:       return UnownedStringSlice::fromLiteral("Clang");
+        case CompilerType::SNC:         return UnownedStringSlice::fromLiteral("SNC");
+        case CompilerType::GHS:         return UnownedStringSlice::fromLiteral("GHS");
+    }
+}
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompiler::Output !!!!!!!!!!!!!!!!!!!!!!*/
 
@@ -167,6 +193,11 @@ SlangResult GenericCPPCompiler::compile(const CompileOptions& options, Output& o
     return m_parseOutputFunc(exeRes, outOutput);
 }
 
+SlangResult GenericCPPCompiler::calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath)
+{
+    return m_calcModuleFilePathFunc(options, outPath);
+}
+
 /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! CPPCompilerUtil !!!!!!!!!!!!!!!!!!!!!!*/
 
 static CPPCompiler::Desc _calcCompiledWithDesc()
@@ -308,7 +339,7 @@ static void _addGCCFamilyCompiler(const String& exeName, CPPCompilerSet* compile
     CPPCompiler::Desc desc;
     if (SLANG_SUCCEEDED(GCCCompilerUtil::calcVersion(exeName, desc)))
     {
-        RefPtr<CPPCompiler> compiler(new GenericCPPCompiler(desc, exeName, &GCCCompilerUtil::calcArgs, &GCCCompilerUtil::parseOutput));
+        RefPtr<CPPCompiler> compiler(new GenericCPPCompiler(desc, exeName, &GCCCompilerUtil::calcArgs, &GCCCompilerUtil::parseOutput, GCCCompilerUtil::calcModuleFilePath));
         compilerSet->addCompiler(compiler);
     }
 }
@@ -377,6 +408,19 @@ void CPPCompilerSet::getCompilers(List<CPPCompiler*>& outCompilers) const
     outCompilers.addRange((CPPCompiler*const*)m_compilers.begin(), m_compilers.getCount());
 }
 
+bool CPPCompilerSet::hasCompiler(CPPCompiler::CompilerType compilerType) const
+{
+    for (CPPCompiler* compiler : m_compilers)
+    {
+        const auto& desc = compiler->getDesc();
+        if (desc.type == compilerType)
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
 void CPPCompilerSet::addCompiler(CPPCompiler* compiler)
 {
     const Index index = _findIndex(compiler->getDesc());

source/core/slang-cpp-compiler.h

@@ -40,6 +40,8 @@ class CPPCompiler: public RefObject
             /// Get the version as a value
         Int getVersionValue() const { return majorVersion * 100 + minorVersion;  }
 
+        void appendAsText(StringBuilder& out) const;
+
             /// Ctor
         Desc(CompilerType inType = CompilerType::Unknown, Int inMajorVersion = 0, Int inMinorVersion = 0):type(inType), majorVersion(inMajorVersion), minorVersion(inMinorVersion) {}
 
@@ -50,16 +52,26 @@ class CPPCompiler: public RefObject
 
     enum class OptimizationLevel
     {
-        Normal,             ///< Normal optimization
-        Debug,              ///< General has no optimizations
+        None,           ///< Don't optimize at all. 
+        Default,        ///< Default optimization level: balance code quality and compilation time. 
+        High,           ///< Optimize aggressively. 
+        Maximal,        ///< Include optimizations that may take a very long time, or may involve severe space-vs-speed tradeoffs 
     };
 
-    enum DebugInfoType
+    enum class DebugInfoType
+    {
+        None,       ///< Don't emit debug information at all. 
+        Minimal,    ///< Emit as little debug information as possible, while still supporting stack traces. 
+        Standard,   ///< Emit whatever is the standard level of debug information for each target. 
+        Maximal,    ///< Emit as much debug information as possible for each target. 
+    };
+    enum class FloatingPointMode
     {
-        None,               ///< Binary has no debug information
-        Maximum,            ///< Has maximum debug information
-        Normal,             ///< Has normal debug information
+        Default, 
+        Fast,
+        Precise,
     };
+
     enum TargetType
     {
         Executable,         ///< Produce an executable
@@ -84,10 +96,11 @@ class CPPCompiler: public RefObject
             };
         };
 
-        OptimizationLevel optimizationLevel = OptimizationLevel::Debug;
-        DebugInfoType debugInfoType = DebugInfoType::Normal;
+        OptimizationLevel optimizationLevel = OptimizationLevel::Default;
+        DebugInfoType debugInfoType = DebugInfoType::Standard;
         TargetType targetType = TargetType::Executable;
         SourceType sourceType = SourceType::CPP;
+        FloatingPointMode floatingPointMode = FloatingPointMode::Default;
 
         Flags flags = Flag::EnableExceptionHandling;
 
@@ -163,6 +176,11 @@ class CPPCompiler: public RefObject
     const Desc& getDesc() const { return m_desc;  }
         /// Compile using the specified options. The result is in resOut
     virtual SlangResult compile(const CompileOptions& options, Output& outOutput) = 0;
+        /// Given the compilation options and the module name, determines the actual file name used for output
+    virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) = 0;
+
+        /// Return the compiler type as name
+    static UnownedStringSlice getCompilerTypeAsText(CompilerType type);
 
 protected:
 
@@ -180,26 +198,31 @@ class GenericCPPCompiler : public CPPCompiler
 
     typedef void(*CalcArgsFunc)(const CPPCompiler::CompileOptions& options, CommandLine& cmdLine);
     typedef SlangResult(*ParseOutputFunc)(const ExecuteResult& exeResult, Output& output);
+    typedef SlangResult(*CalcModuleFilePathFunc)(const CPPCompiler::CompileOptions& options, StringBuilder& outPath);
 
     virtual SlangResult compile(const CompileOptions& options, Output& outOutput) SLANG_OVERRIDE;
+    virtual SlangResult calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath) SLANG_OVERRIDE;
 
-    GenericCPPCompiler(const Desc& desc, const String& exeName, CalcArgsFunc calcArgsFunc, ParseOutputFunc parseOutputFunc) :
+    GenericCPPCompiler(const Desc& desc, const String& exeName, CalcArgsFunc calcArgsFunc, ParseOutputFunc parseOutputFunc, CalcModuleFilePathFunc calcModuleFilePathFunc) :
         Super(desc),
         m_calcArgsFunc(calcArgsFunc),
-        m_parseOutputFunc(parseOutputFunc)
+        m_parseOutputFunc(parseOutputFunc),
+        m_calcModuleFilePathFunc(calcModuleFilePathFunc)
     {
         m_cmdLine.setExecutableFilename(exeName);
     }
 
-    GenericCPPCompiler(const Desc& desc, const CommandLine& cmdLine, CalcArgsFunc calcArgsFunc, ParseOutputFunc parseOutputFunc) :
+    GenericCPPCompiler(const Desc& desc, const CommandLine& cmdLine, CalcArgsFunc calcArgsFunc, ParseOutputFunc parseOutputFunc, CalcModuleFilePathFunc calcModuleFilePathFunc) :
         Super(desc),
         m_cmdLine(cmdLine),
         m_calcArgsFunc(calcArgsFunc),
-        m_parseOutputFunc(parseOutputFunc)
+        m_parseOutputFunc(parseOutputFunc),
+        m_calcModuleFilePathFunc(calcModuleFilePathFunc)
     {}
 
     CalcArgsFunc m_calcArgsFunc;
     ParseOutputFunc m_parseOutputFunc;
+    CalcModuleFilePathFunc m_calcModuleFilePathFunc;
     CommandLine m_cmdLine;
 };
 
@@ -225,6 +248,9 @@ class CPPCompilerSet : public RefObject
         /// Set the default compiler
     void setDefaultCompiler(CPPCompiler* compiler) { m_defaultCompiler = compiler;  }
 
+        /// True if has a compiler of the specified type
+    bool hasCompiler(CPPCompiler::CompilerType compilerType) const;
+
 protected:
 
     Index _findIndex(const CPPCompiler::Desc& desc) const;
@@ -263,6 +289,9 @@ struct CPPCompilerUtil
 
         /// Given a set, registers compilers found through standard means and determines a reasonable default compiler if possible
     static SlangResult initializeSet(CPPCompilerSet* set);
+
+    
+    
 };
 
 

source/core/slang-gcc-compiler-util.cpp

@@ -335,6 +335,34 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
     return SLANG_OK;
 }
 
+/* static */ SlangResult GCCCompilerUtil::calcModuleFilePath(const CompileOptions& options, StringBuilder& outPath)
+{
+    outPath.Clear();
+
+    switch (options.targetType)
+    {
+        case TargetType::SharedLibrary:
+        {
+            outPath << SharedLibrary::calcPlatformPath(options.modulePath.getUnownedSlice());
+            return SLANG_OK;
+        }
+        case TargetType::Executable:
+        {
+            outPath << options.modulePath;
+            outPath << ProcessUtil::getExecutableSuffix();
+            return SLANG_OK;
+        }
+        case TargetType::Object:
+        {
+            // Will be .o for typical gcc targets
+            outPath << options.modulePath << ".o";
+            return SLANG_OK;
+        }
+    }
+
+    return SLANG_FAIL;
+}
+
 /* static */void GCCCompilerUtil::calcArgs(const CompileOptions& options, CommandLine& cmdLine)
 {
     cmdLine.addArg("-fvisibility=hidden");
@@ -349,17 +377,27 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
 
     switch (options.optimizationLevel)
     {
-        case OptimizationLevel::Debug:
+        case OptimizationLevel::None:
         {
             // No optimization
             cmdLine.addArg("-O0");
             break;
         }
-        case OptimizationLevel::Normal:
+        case OptimizationLevel::Default:
         {
             cmdLine.addArg("-Os");
             break;
         }
+        case OptimizationLevel::High:
+        {
+            cmdLine.addArg("-O2");
+            break;
+        }
+        case OptimizationLevel::Maximal:
+        {
+            cmdLine.addArg("-O4");
+            break;
+        }
         default: break;
     }
 
@@ -368,6 +406,29 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
         cmdLine.addArg("-g");
     }
 
+    switch (options.floatingPointMode)
+    {
+        case FloatingPointMode::Default: break;
+        case FloatingPointMode::Precise:
+        {
+            //cmdLine.addArg("-fno-unsafe-math-optimizations");
+            break;
+        }
+        case FloatingPointMode::Fast:
+        {
+            // We could enable SSE with -mfpmath=sse
+            // But that would only make sense on a x64/x86 type processor and only if that feature is present (it is on all x64)
+            cmdLine.addArg("-ffast-math");
+            break;
+        }
+    }
+
+    StringBuilder moduleFilePath;
+    calcModuleFilePath(options, moduleFilePath);
+
+    cmdLine.addArg("-o");
+    cmdLine.addArg(moduleFilePath);
+
     switch (options.targetType)
     {
         case TargetType::SharedLibrary:
@@ -376,22 +437,10 @@ static SlangResult _parseGCCFamilyLine(const UnownedStringSlice& line, LineParse
             cmdLine.addArg("-shared");
             // Position independent
             cmdLine.addArg("-fPIC");
-
-            String sharedLibraryPath = SharedLibrary::calcPlatformPath(options.modulePath.getUnownedSlice());
-
-            cmdLine.addArg("-o");
-            cmdLine.addArg(sharedLibraryPath);
             break;
         }
         case TargetType::Executable:
         {
-            cmdLine.addArg("-o");
-
-            StringBuilder builder;
-            builder << options.modulePath;
-            builder << ProcessUtil::getExecutableSuffix();
-
-            cmdLine.addArg(options.modulePath);
             break;
         }
         case TargetType::Object: