Support include for pulling file into the current module. (shader-slang#3377)

* Support `include` for pulling file into the current module.

* Add auto-completion, hover info and goto-def support.

* Disable warning for missing `module` declaration for now.

---------

Co-authored-by: Yong He <yhe@nvidia.com>

Author: csyonghe

source/compiler-core/slang-include-system.cpp

@@ -113,33 +113,33 @@ SlangResult IncludeSystem::findFile(String const& pathToInclude, String const& p
     return SLANG_E_NOT_FOUND;
 }
 
-SlangResult IncludeSystem::loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>& outBlob)
+SlangResult IncludeSystem::loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>& outBlob, SourceFile*& outSourceFile)
 {
     if (m_sourceManager)
     {
         // See if this an already loaded source file
-        SourceFile* sourceFile = m_sourceManager->findSourceFileRecursively(pathInfo.uniqueIdentity);
+        outSourceFile = m_sourceManager->findSourceFileRecursively(pathInfo.uniqueIdentity);
 
         // If not create a new one, and add to the list of known source files
-        if (!sourceFile)
+        if (!outSourceFile)
         {
             ComPtr<ISlangBlob> foundSourceBlob;
             if (SLANG_FAILED(m_fileSystemExt->loadFile(pathInfo.foundPath.getBuffer(), foundSourceBlob.writeRef())))
             {
                 return SLANG_E_CANNOT_OPEN;
             }
 
-            sourceFile = m_sourceManager->createSourceFileWithBlob(pathInfo, foundSourceBlob);
-            m_sourceManager->addSourceFile(pathInfo.uniqueIdentity, sourceFile);
+            outSourceFile = m_sourceManager->createSourceFileWithBlob(pathInfo, foundSourceBlob);
+            m_sourceManager->addSourceFile(pathInfo.uniqueIdentity, outSourceFile);
 
             outBlob = foundSourceBlob;
             return SLANG_OK;
         }
         else
         {
-            if (sourceFile->getContentBlob())
+            if (outSourceFile->getContentBlob())
             {
-                outBlob = sourceFile->getContentBlob();
+                outBlob = outSourceFile->getContentBlob();
                 return SLANG_OK;
             }
 
@@ -149,7 +149,7 @@ SlangResult IncludeSystem::loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>
                 return SLANG_E_CANNOT_OPEN;
             }
 
-            sourceFile->setContents(foundSourceBlob);
+            outSourceFile->setContents(foundSourceBlob);
 
             outBlob = foundSourceBlob;
             return SLANG_OK;
@@ -158,6 +158,7 @@ SlangResult IncludeSystem::loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>
     else
     {
         // If we don't have the source manager, just load
+        outSourceFile = nullptr;
         return m_fileSystemExt->loadFile(pathInfo.foundPath.getBuffer(), outBlob.writeRef());
     }
 }

source/compiler-core/slang-include-system.h

@@ -36,7 +36,12 @@ struct IncludeSystem
     SlangResult findFile(const String& pathToInclude, const String& pathIncludedFrom, PathInfo& outPathInfo);
     SlangResult findFile(SlangPathType fromPathType, const String& fromPath, const String& path, PathInfo& outPathInfo);
     String simplifyPath(const String& path);
-    SlangResult loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>& outBlob);
+    SlangResult loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>& outBlob, SourceFile*& outSourceFile);
+    inline SlangResult loadFile(const PathInfo& pathInfo, ComPtr<ISlangBlob>& outBlob)
+    {
+        SourceFile* sourceFile;
+        return loadFile(pathInfo, outBlob, sourceFile);
+    }
 
     SlangResult findAndLoadFile(const String& pathToInclude, const String& pathIncludedFrom, PathInfo& outPathInfo, ComPtr<ISlangBlob>& outBlob);
 
@@ -45,6 +50,7 @@ struct IncludeSystem
     SourceManager* getSourceManager() const { return m_sourceManager; }
 
         /// Ctor
+    IncludeSystem() = default;
     IncludeSystem(SearchDirectoryList* searchDirectories, ISlangFileSystemExt* fileSystemExt, SourceManager* sourceManager = nullptr);
 
 protected:

source/compiler-core/slang-source-loc.cpp

@@ -919,6 +919,13 @@ void SourceManager::addSourceFile(const String& uniqueIdentity, SourceFile* sour
     m_sourceFileMap.add(uniqueIdentity, sourceFile);
 }
 
+void SourceManager::addSourceFileIfNotExist(const String& uniqueIdentity, SourceFile* sourceFile)
+{
+    if (findSourceFileRecursively(uniqueIdentity))
+        return;
+    m_sourceFileMap.addIfNotExists(uniqueIdentity, sourceFile);
+}
+
 HumaneSourceLoc SourceManager::getHumaneLoc(SourceLoc loc, SourceLocType type)
 {
     SourceView* sourceView = findSourceViewRecursively(loc);

source/compiler-core/slang-source-loc.h

@@ -482,6 +482,7 @@ struct SourceManager
 
         /// Add a source file, uniqueIdentity must be unique for this manager AND any parents
     void addSourceFile(const String& uniqueIdentity, SourceFile* sourceFile);
+    void addSourceFileIfNotExist(const String& uniqueIdentity, SourceFile* sourceFile);
 
         /// Get the slice pool
     StringSlicePool& getStringSlicePool() { return m_slicePool; }

source/core/slang-io.cpp

@@ -794,6 +794,17 @@ namespace Slang
     }
 #endif
 
+    bool Path::equals(String path1, String path2)
+    {
+        Path::getCanonical(path1, path1);
+        Path::getCanonical(path2, path2);
+#if SLANG_WINDOWS_FAMILY
+        return path1.getUnownedSlice().caseInsensitiveEquals(path2.getUnownedSlice());
+#else
+        return path1 == path2;
+#endif
+    }
+
     /// Gets the path to the executable that was invoked that led to the current threads execution
     /// If run from a shared library/dll will be the path of the executable that loaded said library
     /// @param outPath Pointer to buffer to hold the path.

source/core/slang-io.h

@@ -211,6 +211,8 @@ namespace Slang
             /// @param path
             /// @return SLANG_OK if file or directory is removed
         static SlangResult remove(const String& path);
+
+        static bool equals(String path1, String path2);
     };
 
     struct URI

source/slang/slang-ast-decl.h

@@ -436,6 +436,12 @@ class ModuleDecl : public NamespaceDeclBase
 
 };
 
+// Represents a transparent scope of declarations that are defined in a single source file.
+class FileDecl : public ContainerDecl
+{
+    SLANG_AST_CLASS(FileDecl);
+};
+
     /// A declaration that brings members of another declaration or namespace into scope
 class UsingDecl : public Decl
 {
@@ -449,15 +455,12 @@ class UsingDecl : public Decl
     Scope* scope = nullptr;
 };
 
-class ImportDecl : public Decl
+class FileReferenceDeclBase : public Decl
 {
-    SLANG_AST_CLASS(ImportDecl)
+    SLANG_AST_CLASS(FileReferenceDeclBase)
 
     // The name of the module we are trying to import
     NameLoc moduleNameAndLoc;
-    
-    // The module that actually got imported
-    ModuleDecl* importedModuleDecl = nullptr;
 
     SourceLoc startLoc;
     SourceLoc endLoc;
@@ -467,6 +470,36 @@ class ImportDecl : public Decl
     Scope* scope = nullptr;
 };
 
+class ImportDecl : public FileReferenceDeclBase
+{
+    SLANG_AST_CLASS(ImportDecl)
+
+    // The module that actually got imported
+    ModuleDecl* importedModuleDecl = nullptr;
+};
+
+class IncludeDeclBase : public FileReferenceDeclBase
+{
+    SLANG_AST_CLASS(IncludeDeclBase)
+
+    FileDecl* fileDecl = nullptr;
+};
+
+class IncludeDecl : public IncludeDeclBase
+{
+    SLANG_AST_CLASS(IncludeDecl)
+};
+
+class ImplementingDecl : public IncludeDeclBase
+{
+    SLANG_AST_CLASS(ImplementingDecl)
+};
+
+class ModuleDeclarationDecl : public Decl
+{
+    SLANG_AST_CLASS(ModuleDeclarationDecl);
+};
+
 // A generic declaration, parameterized on types/values
 class GenericDecl : public ContainerDecl
 {