Improvements around Visual Studio versions/matching versions (shader-slang#2267)

* #include an absolute path didn't work - because paths were taken to always be relative.

* Use TerminatedUnownedStringSlice for literals in output C++.

* Remove Escape/Unescape functions used in slang-token-reader.cpp
Add target type of 'host-cpp' etc to map to the target types.

* Fix some corner cases around string encoding.

* Added unit test for string escaping.
Fixed some assorted escaping bugs.

* Updated test output.

* Added decode test.

* Stop using hex output, to get around 'greedy' aspect. Use octal instead.

* Added HostHostCallable
Small changes to use ArtifactDesc/Info instead of large switches.

* Fix C++ emit to handle arbitrary function export.

* Add options handling for callable without an output being specified.

* Can compile with COM interface. Added example using com interface.

* Use the IR Ptr type instead of hack in C++ emit for interfaces.

* Fix issue with outputting the COM call when ptr is used.

* Fix crash issue on compilation failure.

* Add support for __global.

* Added `ActualGlobalRate`
Added special handling around globals and COM interfaces.
Tested out in cpu-com-example.

* Fix typo in NodeBase.

* Support for accessing globals by name working.

* Check that actual global initialization is working.

* Refactor the com replacement such that it doesn't need a cache or do anything special with GlobalVar.

* Remove context.
Only create replacement if needed.

* Split out COM host-callable into a unit-test.

* host-callable com testing on C++and llvm.

* Comment around the COM ptr replacement.

* Disable com test on vs 32 bit.
Fix C++ prelude

* Disable 32 bit targets testing com host-callable.

* Use JSON parsing to locate VS version.

* Need platform detection in C++prelude.

* Fix com host callable test for LLVM.

* WIP improments finding downstream compiler version.

* Work around for not being able to include "targetConditionals.h"

* Matching semantic versioning support.

* DownstreamMatchVersion -> DownstreamCompilerMatchVersion
Small improvements.

Author: jsmall-zzz

source/compiler-core/slang-downstream-compiler.cpp

@@ -519,34 +519,32 @@ SlangResult CommandLineDownstreamCompiler::compile(const CompileOptions& inOptio
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompiler::Desc !!!!!!!!!!!!!!!!!!!!!!*/
 
-static DownstreamCompiler::Desc _calcCompiledWithDesc()
+static DownstreamCompilerMatchVersion _calcCompiledVersion()
 {
-    DownstreamCompiler::Desc desc;
+    DownstreamCompilerMatchVersion matchVersion;
 
 #if SLANG_VC
-    desc = WinVisualStudioUtil::getDesc(WinVisualStudioUtil::getCompiledVersion());
+    matchVersion = WinVisualStudioUtil::getCompiledVersion();
 #elif SLANG_CLANG
-    desc.type = SLANG_PASS_THROUGH_CLANG;
-    desc.majorVersion = Int(__clang_major__);
-    desc.minorVersion = Int(__clang_minor__);
+    matchVersion.type = SLANG_PASS_THROUGH_CLANG;
+    matchVersion.matchVersion.set(Index(__clang_major__), Index(__clang_minor__));
 #elif SLANG_GCC
-    desc.type = SLANG_PASS_THROUGH_GCC;
-    desc.majorVersion = Int(__GNUC__);
-    desc.minorVersion = Int(__GNUC_MINOR__);
+    matchVersion.type = SLANG_PASS_THROUGH_GCC;
+    matchVersion.matchVersion.set(Index(__GNUC__), Index(__GNUC_MINOR__));
 #else
     // TODO(JS): Hmmm None is not quite the same as unknown. It works for now, but we might want to have a distinct enum for unknown.
-    desc.type = SLANG_PASS_THROUGH_NONE;        
+    matchVersion.type = SLANG_PASS_THROUGH_NONE;
 #endif
 
-    return desc;
+    return matchVersion;
 }
 
 /* !!!!!!!!!!!!!!!!!!!!!!!!! DownstreamCompilerUtil !!!!!!!!!!!!!!!!!!!!!!*/
 
-const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc()
+DownstreamCompilerMatchVersion DownstreamCompilerUtil::getCompiledVersion()
 {
-    static DownstreamCompiler::Desc s_desc = _calcCompiledWithDesc();
-    return s_desc;
+    static DownstreamCompilerMatchVersion s_version = _calcCompiledVersion();
+    return s_version;
 }
 
 /* static */DownstreamCompiler* DownstreamCompilerUtil::findCompiler(const DownstreamCompilerSet* set, MatchType matchType, const DownstreamCompiler::Desc& desc)
@@ -626,22 +624,70 @@ const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc()
     return (bestIndex >= 0) ? compilers[bestIndex] : nullptr;
 }
 
-/* static */DownstreamCompiler* DownstreamCompilerUtil::findClosestCompiler(const List<DownstreamCompiler*>& compilers, const DownstreamCompiler::Desc& desc)
+/* static */DownstreamCompiler* DownstreamCompilerUtil::findCompiler(const List<DownstreamCompiler*>& compilers, const DownstreamCompiler::Desc& desc)
 {
-    DownstreamCompiler* compiler;
+    for (auto compiler : compilers)
+    {
+        if (compiler->getDesc() == desc)
+        {
+            return compiler;
+        }
+    }
+    return nullptr;
+}
 
-    compiler = findCompiler(compilers, MatchType::MinGreaterEqual, desc);
-    if (compiler)
+/* static */DownstreamCompiler* DownstreamCompilerUtil::findCompiler(const List<DownstreamCompiler*>& compilers, SlangPassThrough type, const SemanticVersion& version)
+{
+    DownstreamCompiler::Desc desc;
+    desc.type = type;
+    desc.majorVersion = version.m_major;
+    desc.minorVersion = version.m_minor;
+    return findCompiler(compilers, desc);
+}
+
+/* static */void DownstreamCompilerUtil::findVersions(const List<DownstreamCompiler*>& compilers, SlangPassThrough type, List<SemanticVersion>& outVersions)
+{
+    for (auto compiler : compilers)
     {
-        return compiler;
+        auto desc = compiler->getDesc();
+
+        if (desc.type == type)
+        {
+            outVersions.add(SemanticVersion(int(desc.majorVersion), int(desc.minorVersion), 0));
+        }
     }
-    compiler = findCompiler(compilers, MatchType::MinAbsolute, desc);
-    if (compiler)
+}
+
+/* static */DownstreamCompiler* DownstreamCompilerUtil::findClosestCompiler(const List<DownstreamCompiler*>& compilers, const DownstreamCompilerMatchVersion& matchVersion)
+{
+    List<SemanticVersion> versions;
+
+    findVersions(compilers, matchVersion.type, versions);
+
+    if (versions.getCount() > 0)
     {
-        return compiler;
+        if (versions.getCount() == 1)
+        {
+            // Must be that one
+            return findCompiler(compilers, matchVersion.type, versions[0]);
+        }
+
+        // Okay lets find the best one
+        auto bestVersion = MatchSemanticVersion::findAnyBest(versions.getBuffer(), versions.getCount(), matchVersion.matchVersion);
+
+        // If one is found use it
+        if (bestVersion.isSet())
+        {
+            return findCompiler(compilers, matchVersion.type, bestVersion);
+        }
     }
 
     {
+        // TODO(JS):
+        // NOTE! This may not really be appropriate, because LLVM is *not* interchangable with 
+        // a 'normal' C++ compiler as cannot access standard libraries/headers.
+        // So `slang-llvm` can't be used for 'host' code.
+
         // These compilers should be usable interchangably. The order is important, as the first one that matches will
         // be used, so LLVM is used before CLANG or GCC if appropriate
         const SlangPassThrough compatiblePassThroughs[] =
@@ -651,31 +697,20 @@ const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc()
             SLANG_PASS_THROUGH_GCC,
         };
 
-        bool isCompatible = false;
-        for (auto passThrough : compatiblePassThroughs)
-        {
-            if (desc.type == passThrough)
-            {
-                isCompatible = true;
-                break;
-            }
-        }
-
-        if (isCompatible)
+        // Check the version is one of the compatible types
+        if (makeConstArrayView(compatiblePassThroughs).indexOf(matchVersion.type) >= 0)
         {
+            // Try each compatible type in turn
             for (auto passThrough : compatiblePassThroughs)
             {
-                if (passThrough != desc.type)
+                versions.clear();
+                findVersions(compilers, passThrough, versions);
+
+                if (versions.getCount() > 0)
                 {
-                    DownstreamCompiler::Desc compatible;
-                    
-                    compatible.type = passThrough;
-                    // Find the latest version.
-                    compiler = findCompiler(compilers, MatchType::Newest, compatible);
-                    if (compiler)
-                    {
-                        return compiler;
-                    }
+                    // Get the latest version (as we have no way to really compare)
+                    auto latestVersion = SemanticVersion::getLatest(versions.getBuffer(), versions.getCount());
+                    return findCompiler(compilers, matchVersion.type, latestVersion);
                 }
             }
         }
@@ -684,16 +719,11 @@ const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc()
     return nullptr;
 }
 
-/* static */DownstreamCompiler* DownstreamCompilerUtil::findClosestCompiler(const DownstreamCompilerSet* set, const DownstreamCompiler::Desc& desc)
+/* static */DownstreamCompiler* DownstreamCompilerUtil::findClosestCompiler(const DownstreamCompilerSet* set, const DownstreamCompilerMatchVersion& matchVersion)
 {
-    DownstreamCompiler* compiler = set->getCompiler(desc);
-    if (compiler)
-    {
-        return compiler;
-    }
     List<DownstreamCompiler*> compilers;
     set->getCompilers(compilers);
-    return findClosestCompiler(compilers, desc);
+    return findClosestCompiler(compilers, matchVersion);
 }
 
 /* static */void DownstreamCompilerUtil::updateDefault(DownstreamCompilerSet* set, SlangSourceLanguage sourceLanguage)
@@ -708,7 +738,7 @@ const DownstreamCompiler::Desc& DownstreamCompilerUtil::getCompiledWithDesc()
             // Find the compiler closest to the compiler this was compiled with
             if (!compiler)
             {
-                compiler = findClosestCompiler(set, getCompiledWithDesc());
+                compiler = findClosestCompiler(set, getCompiledVersion());
             }
             break;
         }

source/compiler-core/slang-downstream-compiler.h

@@ -154,6 +154,21 @@ class BlobDownstreamCompileResult : public DownstreamCompileResult
     ComPtr<ISlangBlob> m_blob;
 };
 
+// Combination of a downstream compiler type (pass through) and 
+// a match version.
+struct DownstreamCompilerMatchVersion
+{
+    DownstreamCompilerMatchVersion(SlangPassThrough inType, MatchSemanticVersion inMatchVersion):
+        type(inType),
+        matchVersion(inMatchVersion)
+    {}
+
+    DownstreamCompilerMatchVersion():type(SLANG_PASS_THROUGH_NONE) {}
+
+    SlangPassThrough type;                  ///< The type of the compiler
+    MatchSemanticVersion matchVersion;      ///< The match version
+};
+
 class DownstreamCompiler: public RefObject
 {
 public:
@@ -190,6 +205,8 @@ class DownstreamCompiler: public RefObject
         Info infos[int(SLANG_PASS_THROUGH_COUNT_OF)];
     };
 
+    
+    // Compiler description
     struct Desc
     {
         typedef Desc ThisType;
@@ -208,11 +225,17 @@ class DownstreamCompiler: public RefObject
             /// Ctor
         explicit Desc(SlangPassThrough inType = SLANG_PASS_THROUGH_NONE, Int inMajorVersion = 0, Int inMinorVersion = 0):type(inType), majorVersion(inMajorVersion), minorVersion(inMinorVersion) {}
 
+        explicit Desc(SlangPassThrough inType, const SemanticVersion& version):type(inType), majorVersion(version.m_major), minorVersion(version.m_minor) {}
+
         SlangPassThrough type;      ///< The type of the compiler
+
+        /// TODO(JS): Would probably be better if changed to SemanticVersion, but not trivial to change
+        // because this type is part of the DownstreamCompiler interface, which is used with `slang-llvm`.
         Int majorVersion;           ///< Major version (interpretation is type specific)
         Int minorVersion;           ///< Minor version (interpretation is type specific)
     };
 
+
     enum class OptimizationLevel
     {
         None,           ///< Don't optimize at all. 
@@ -526,12 +549,19 @@ struct DownstreamCompilerUtil: public DownstreamCompilerBaseUtil
     static DownstreamCompiler* findCompiler(const DownstreamCompilerSet* set, MatchType matchType, const DownstreamCompiler::Desc& desc);
     static DownstreamCompiler* findCompiler(const List<DownstreamCompiler*>& compilers, MatchType matchType, const DownstreamCompiler::Desc& desc);
 
+    static DownstreamCompiler* findCompiler(const List<DownstreamCompiler*>& compilers, SlangPassThrough type, const SemanticVersion& version);
+    static DownstreamCompiler* findCompiler(const List<DownstreamCompiler*>& compilers, const DownstreamCompiler::Desc& desc);
+
+        /// Find all the compilers with the version
+    static void findVersions(const List<DownstreamCompiler*>& compilers, SlangPassThrough compiler, List<SemanticVersion>& versions);
+
+    
         /// Find the compiler closest to the desc 
-    static DownstreamCompiler* findClosestCompiler(const List<DownstreamCompiler*>& compilers, const DownstreamCompiler::Desc& desc);
-    static DownstreamCompiler* findClosestCompiler(const DownstreamCompilerSet* set, const DownstreamCompiler::Desc& desc);
+    static DownstreamCompiler* findClosestCompiler(const List<DownstreamCompiler*>& compilers, const DownstreamCompilerMatchVersion& version);
+    static DownstreamCompiler* findClosestCompiler(const DownstreamCompilerSet* set, const DownstreamCompilerMatchVersion& version);
 
         /// Get the information on the compiler used to compile this source
-    static const DownstreamCompiler::Desc& getCompiledWithDesc();
+    static DownstreamCompilerMatchVersion getCompiledVersion();
 
     static void updateDefault(DownstreamCompilerSet* set, SlangSourceLanguage sourceLanguage);
     static void updateDefaults(DownstreamCompilerSet* set);