Add slangc interface to compile and use ir modules. (shader-slang#3615)

* Add slangc interface to compile and use ir modules.

* Fix glsl scalar layout settings not copied to target.

* Fix.

* Cleanups.

Author: csyonghe

examples/platform-test/main.cpp

@@ -25,12 +25,12 @@ void onLostFocus()
 
 void onKeyDown(platform::KeyEventArgs args)
 {
-    printf("onKeyDown(key=0x%02x, buttons=0x%02x)\n", args.key, args.buttons);
+    printf("onKeyDown(key=0x%02x, buttons=0x%02x)\n", (uint32_t)args.key, args.buttons);
 }
 
 void onKeyUp(platform::KeyEventArgs args)
 {
-    printf("okKeyUp(key=0x%02x, buttons=0x%02x)\n", args.key, args.buttons);
+    printf("okKeyUp(key=0x%02x, buttons=0x%02x)\n", (uint32_t)args.key, args.buttons);
 }
 
 void onKeyPress(platform::KeyEventArgs args)

slang.h

@@ -855,7 +855,6 @@ extern "C"
             ReportDownstreamTime,  // bool
             ReportPerfBenchmark,   // bool
             SkipSPIRVValidation,   // bool
-
             SourceEmbedStyle,
             SourceEmbedName,
             SourceEmbedLanguage,
@@ -1845,6 +1844,7 @@ extern "C"
     /*! @see slang::ICompileRequest::addLibraryReference */
     SLANG_API SlangResult spAddLibraryReference(
         SlangCompileRequest*    request,
+        const char* basePath,
         const void* libData,
         size_t libDataSize);
 
@@ -3952,10 +3952,12 @@ namespace slang
             /** Add a slang library - such that its contents can be referenced during linking.
             This is equivalent to the -r command line option.
 
+            @param basePath The base path used to lookup referenced modules.
             @param libData The library data
             @param libDataSize The size of the library data
             */
         virtual SLANG_NO_THROW SlangResult SLANG_MCALL addLibraryReference(
+            const char* basePath,
             const void* libData,
             size_t libDataSize) = 0;
 

source/compiler-core/slang-source-loc.h

@@ -61,7 +61,7 @@ struct PathInfo
         /// True if has a canonical path
     SLANG_FORCE_INLINE bool hasUniqueIdentity() const { return type == Type::Normal && uniqueIdentity.getLength() > 0; }
         /// True if has a regular found path
-    SLANG_FORCE_INLINE bool hasFoundPath() const { return type == Type::Normal || type == Type::FoundPath || (type == Type::FromString && foundPath.getLength() > 0); }
+    SLANG_FORCE_INLINE bool hasFoundPath() const { return (type == Type::Normal || type == Type::FoundPath || type == Type::FromString) && foundPath.getLength() > 0; }
         /// True if has a found path that has originated from a file (as opposed to string or some other origin)
     SLANG_FORCE_INLINE bool hasFileFoundPath() const { return (type == Type::Normal || type == Type::FoundPath) && foundPath.getLength() > 0; }
         /// Get the 'name'/path of the item. Will return an empty string if not applicable or not set.

source/slang/slang-api.cpp

@@ -495,11 +495,12 @@ SLANG_API void spSetDefaultModuleName(
 
 SLANG_API SlangResult spAddLibraryReference(
     slang::ICompileRequest*    request,
+    const char* basePath,
     const void* libData,
     size_t libDataSize)
 {
     SLANG_ASSERT(request);
-    return request->addLibraryReference(libData, libDataSize);
+    return request->addLibraryReference(basePath, libData, libDataSize);
 }
 
 SLANG_API void spTranslationUnit_addPreprocessorDefine(

source/slang/slang-check-decl.cpp

@@ -1712,7 +1712,7 @@ namespace Slang
         {
             if (auto varDeclRefType = as<DeclRefType>(varDecl->type.type))
             {
-                parentAggTypeDecl->unionTagsWith(getTypeTags(varDecl->type.type));
+                parentAggTypeDecl->unionTagsWith(getTypeTags(varDeclRefType));
             }
         }
 
@@ -9321,20 +9321,21 @@ namespace Slang
             else if (as<PrivateModifier>(modifier))
                 return DeclVisibility::Private;
         }
-
         // Interface members will always have the same visibility as the interface itself.
         if (auto interfaceDecl = findParentInterfaceDecl(decl))
         {
             return getDeclVisibility(interfaceDecl);
         }
-        else if (as<NamespaceDecl>(decl))
+        auto defaultVis = DeclVisibility::Default;
+        if (auto parentModule = getModuleDecl(decl))
+            defaultVis = parentModule->isInLegacyLanguage ? DeclVisibility::Public : DeclVisibility::Internal;
+
+        // Members of other agg type decls will have their default visibility capped to the parents'.
+        if (as<NamespaceDecl>(decl))
         {
             return DeclVisibility::Public;
         }
-        if (auto parentModule = getModuleDecl(decl))
-            return parentModule->isInLegacyLanguage ? DeclVisibility::Public : DeclVisibility::Internal;
-
-        return DeclVisibility::Default;
+        return defaultVis;
     }
 
     void diagnoseCapabilityProvenance(DiagnosticSink* sink, Decl* decl, CapabilityAtom missingAtom)

source/slang/slang-check-modifier.cpp

@@ -989,7 +989,6 @@ namespace Slang
         case ASTNodeType::PreciseModifier:
         case ASTNodeType::IntrinsicOpModifier:
         case ASTNodeType::InlineModifier:
-        case ASTNodeType::ExternModifier:
         case ASTNodeType::HLSLExportModifier:
         case ASTNodeType::ExternCppModifier:
         case ASTNodeType::ExportedModifier:

source/slang/slang-compiler.cpp

@@ -1488,7 +1488,7 @@ namespace Slang
             // Add all of the module libraries
             libraries.addRange(linkage->m_libModules.getBuffer(), linkage->m_libModules.getCount());
         }
-        
+
         options.compilerSpecificArguments = allocator.allocate(compilerSpecificArguments);
         options.requiredCapabilityVersions = SliceUtil::asSlice(requiredCapabilityVersions);
         options.libraries = SliceUtil::asSlice(libraries);
@@ -1879,14 +1879,7 @@ namespace Slang
         SerialContainerUtil::WriteOptions options;
 
         options.compressionType = linkage->m_optionSet.getEnumOption<SerialCompressionType>(CompilerOptionName::IrCompression);
-        if (linkage->m_optionSet.getBoolOption(CompilerOptionName::Obfuscate))
-        {
-            // If code is obfuscated, we *disable* AST output as it is not obfuscated and will reveal
-            // too much about IR.
-            // Also currently only IR is needed.
-            options.optionFlags &= ~SerialOptionFlag::ASTModule;
-        }
-        
+
         // If debug information is enabled, enable writing out source locs
         if (_shouldWriteSourceLocs(linkage))
         {

source/slang/slang-compiler.h

@@ -25,12 +25,10 @@
 
 #include "slang-capability.h"
 #include "slang-diagnostics.h"
-
 #include "slang-preprocessor.h"
 #include "slang-profile.h"
 #include "slang-syntax.h"
 #include "slang-content-assist-info.h"
-
 #include "slang-hlsl-to-vulkan-layout-options.h"
 #include "slang-compiler-options.h"
 #include "slang-serialize-ir-types.h"
@@ -1475,7 +1473,7 @@ namespace Slang
 
         RefPtr<EntryPoint> findEntryPointByName(UnownedStringSlice const& name);
 
-        List<RefPtr<EntryPoint>> const& getEntryPoints() { return m_entryPoints; }
+        List<RefPtr<EntryPoint>>& getEntryPoints() { return m_entryPoints; }
         void _addEntryPoint(EntryPoint* entryPoint);
         void _processFindDeclsExportSymbolsRec(Decl* decl);
 
@@ -1551,6 +1549,8 @@ namespace Slang
     public:
         TranslationUnitRequest(
             FrontEndCompileRequest* compileRequest);
+        TranslationUnitRequest(
+            FrontEndCompileRequest* compileRequest, Module* m);
 
         // The parent compile request
         FrontEndCompileRequest* compileRequest = nullptr;
@@ -1596,6 +1596,8 @@ namespace Slang
             /// Result of compiling this translation unit (a module)
         RefPtr<Module> module;
 
+        bool isChecked = false;
+
         Module* getModule() { return module; }
         ModuleDecl* getModuleDecl() { return module->getModuleDecl(); }
 
@@ -1755,6 +1757,8 @@ namespace Slang
         Source, IR
     };
 
+    struct SerialContainerDataModule;
+
         /// A context for loading and re-using code modules.
     class Linkage : public RefObject, public slang::ISession
     {
@@ -1959,6 +1963,11 @@ namespace Slang
             SourceLoc const& loc,
             DiagnosticSink* sink,
             const LoadedModuleDictionary* additionalLoadedModules);
+        RefPtr<Module> loadDeserializedModule(
+            Name* name,
+            const PathInfo& filePathInfo,
+            SerialContainerDataModule& m,
+            DiagnosticSink* sink);
 
         SourceFile* loadSourceFile(String pathFrom, String path);
 
@@ -1979,7 +1988,7 @@ namespace Slang
             DiagnosticSink*     sink,
             const LoadedModuleDictionary* loadedModules = nullptr);
 
-        void prepareDeserializedModule(Module* module, DiagnosticSink* sink);
+        void prepareDeserializedModule(SerialContainerDataModule& moduleEntry, const PathInfo& pathInfo, Module* module, DiagnosticSink* sink);
 
         SourceFile* findFile(Name* name, SourceLoc loc, IncludeSystem& outIncludeSystem);
         struct IncludeResult
@@ -2661,7 +2670,7 @@ namespace Slang
         virtual SLANG_NO_THROW void SLANG_MCALL addTranslationUnitPreprocessorDefine(int translationUnitIndex, const char* key, const char* value) SLANG_OVERRIDE;
         virtual SLANG_NO_THROW void SLANG_MCALL addTranslationUnitSourceFile(int translationUnitIndex, char const* path) SLANG_OVERRIDE;
         virtual SLANG_NO_THROW void SLANG_MCALL addTranslationUnitSourceString(int translationUnitIndex, char const* path, char const* source) SLANG_OVERRIDE;
-        virtual SLANG_NO_THROW SlangResult SLANG_MCALL addLibraryReference(const void* libData, size_t libDataSize) SLANG_OVERRIDE;
+        virtual SLANG_NO_THROW SlangResult SLANG_MCALL addLibraryReference(const char* basePath, const void* libData, size_t libDataSize) SLANG_OVERRIDE;
         virtual SLANG_NO_THROW void SLANG_MCALL addTranslationUnitSourceStringSpan(int translationUnitIndex, char const* path, char const* sourceBegin, char const* sourceEnd) SLANG_OVERRIDE;
         virtual SLANG_NO_THROW void SLANG_MCALL addTranslationUnitSourceBlob(int translationUnitIndex, char const* path, ISlangBlob* sourceBlob) SLANG_OVERRIDE;
         virtual SLANG_NO_THROW int SLANG_MCALL addEntryPoint(int translationUnitIndex, char const* name, SlangStage stage) SLANG_OVERRIDE;

source/slang/slang-ir-link.cpp

@@ -1477,14 +1477,7 @@ LinkedIR linkIR(
     {
         irModules.add(irModule);
     });
-    for (IArtifact* artifact : linkage->m_libModules)
-    {
-        if (auto library = findRepresentation<ModuleLibrary>(artifact))
-        {
-            irModules.addRange(library->m_modules.getBuffer()->readRef(), library->m_modules.getCount());
-        }
-    }
-    
+
     // Add any modules that were loaded as libraries
     for (IRModule* irModule : irModules)
     {

source/slang/slang-module-library.cpp

@@ -39,7 +39,7 @@ void* ModuleLibrary::castAs(const Guid& guid)
     return getObject(guid);
 }
 
-SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outLibrary)
+SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, String path, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outLibrary)
 {
     auto library = new ModuleLibrary;
     ComPtr<IModuleLibrary> scopeLibrary(library);
@@ -51,10 +51,6 @@ SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, EndToEndCo
     SLANG_RETURN_ON_FAIL(RiffUtil::read(&memoryStream, riffContainer));
 
     auto linkage = req->getLinkage();
-
-    // TODO(JS): May be better to have a ITypeComponent that encapsulates a collection of modules
-    // For now just add to the linkage
-
     {
         SerialContainerData containerData;
 
@@ -65,15 +61,29 @@ SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, EndToEndCo
         options.sourceManager = linkage->getSourceManager();
         options.linkage = req->getLinkage();
         options.sink = req->getSink();
-
+        options.astBuilder = linkage->getASTBuilder();
+        options.modulePath = path;
         SLANG_RETURN_ON_FAIL(SerialContainerUtil::read(&riffContainer, options, nullptr, containerData));
+        DiagnosticSink sink;
 
-        for (const auto& module : containerData.modules)
+        // Modules in the container should be serialized in its depedency order,
+        // so that we always load the dependencies before the consuming module.
+        for (auto& module : containerData.modules)
         {
             // If the irModule is set, add it
             if (module.irModule)
             {
-                library->m_modules.add(module.irModule);
+                if (module.dependentFiles.getCount() == 0)
+                    return SLANG_FAIL;
+                if (!module.astRootNode)
+                    return SLANG_FAIL;
+                auto loadedModule = linkage->loadDeserializedModule(
+                    as<ModuleDecl>(module.astRootNode)->getName(),
+                    PathInfo::makePath(module.dependentFiles.getFirst()),
+                    module, &sink);
+                if (!loadedModule)
+                    return SLANG_FAIL;
+                library->m_modules.add(loadedModule);
             }
         }
 
@@ -93,7 +103,7 @@ SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, EndToEndCo
     return SLANG_OK;
 }
 
-SlangResult loadModuleLibrary(ArtifactKeep keep, IArtifact* artifact, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outLibrary)
+SlangResult loadModuleLibrary(ArtifactKeep keep, IArtifact* artifact, String path, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outLibrary)
 {
     if (auto foundLibrary = findRepresentation<IModuleLibrary>(artifact))
     {
@@ -107,7 +117,7 @@ SlangResult loadModuleLibrary(ArtifactKeep keep, IArtifact* artifact, EndToEndCo
 
     // Load the module
     ComPtr<IModuleLibrary> library;
-    SLANG_RETURN_ON_FAIL(loadModuleLibrary((const Byte*)blob->getBufferPointer(), blob->getBufferSize(), req, library));
+    SLANG_RETURN_ON_FAIL(loadModuleLibrary((const Byte*)blob->getBufferPointer(), blob->getBufferSize(), path, req, library));
 
     if (canKeep(keep))
     {

source/slang/slang-module-library.h

@@ -30,16 +30,16 @@ class ModuleLibrary : public ComBaseObject, public IModuleLibrary
     virtual SLANG_NO_THROW bool SLANG_MCALL exists() SLANG_OVERRIDE { return true; }
 
     List<FrontEndCompileRequest::ExtraEntryPointInfo> m_entryPoints;
-    List<RefPtr<IRModule>> m_modules;
+    List<RefPtr<Module>> m_modules;
 
     void* getInterface(const Guid& uuid);
     void* getObject(const Guid& uuid);
 };
 
-SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outModule);
+SlangResult loadModuleLibrary(const Byte* inBytes, size_t bytesCount, String Path, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outModule);
 
 // Given a product make available as a module
-SlangResult loadModuleLibrary(ArtifactKeep keep, IArtifact* artifact, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outModule);
+SlangResult loadModuleLibrary(ArtifactKeep keep, IArtifact* artifact, String Path, EndToEndCompileRequest* req, ComPtr<IModuleLibrary>& outModule);
 
 } // namespace Slang