Feature/serialization debug info (shader-slang#767)

* Remove AppContext. Use StdChannels to hold writers, and TestToolUtil to hold test tool specific functionality.

* StdChannels -> StdWriters

* getStdOut -> getOut, getStdError -> getError

* Renamed main.cpp files of tools to try and stop visual studio getting confused between files - such that clicking on an error takes editor to the right location.

* Work in progress on being able to serialize debug information.

* * Added MemoryStream
* First pass converting to IRSerialData
* Able to read and write IRSerialData with debug data

* Start at reconstruting IR serialized data.

* First pass of generation debug SourceLocs from debug data. Works for test set for line nos.

* Bug fixes.
Moved testing of serialization into IRSerialUtil

* Work around problem with  irModule = generateIRForTranslationUnit(translationUnit); two times in a row produces different output(!). Fix by just creating once.

* Remove problem with use of ternary op in slang.cpp on gcc/clang.

* Added -verify-debug-serial-ir option that makes IR modules go through full serialization with debug information and verification.

* Add a test that does serial debug verification that is run by default on linux.

Author: jsmall-zzz

source/core/slang-string-slice-pool.h

@@ -50,6 +50,8 @@ class StringSlicePool
 
         /// Convert a handle to and index. (A handle is just an index!) 
     static int asIndex(Handle handle) { return int(handle); }
+        /// Returns true if the handle is to a slice that contains characters (ie not null or empty)
+    static bool hasContents(Handle handle) { return int(handle) >= kNumDefaultHandles; }
 
         /// Ctor
     StringSlicePool();

source/core/stream.cpp

@@ -153,6 +153,7 @@ namespace Slang
 			break;
 		case Slang::SeekOrigin::End:
 			_origin = SEEK_END;
+            // JS TODO: This doesn't seem right, the offset can mean it's not at the end
 			endReached = true;
 			break;
 		case Slang::SeekOrigin::Current:
@@ -215,4 +216,79 @@ namespace Slang
 	{
 		return endReached;
 	}
+
+    // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! MemoryStream !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+    void MemoryStream::Seek(SeekOrigin origin, Int64 offset)
+    {
+        Int64 pos = 0;
+        switch (origin)
+        {
+        case Slang::SeekOrigin::Start:
+            pos = offset;
+            break;
+        case Slang::SeekOrigin::End:
+            pos = Int64(m_contents.Count()) + offset;
+            break;
+        case Slang::SeekOrigin::Current:
+            pos = Int64(m_position) + offset;
+            break;
+        default:
+            throw NotSupportedException("Unsupported seek origin.");
+            break;
+        }
+
+        m_atEnd = false;
+
+        // Clamp to the valid range
+        pos = (pos < 0) ? 0 : pos;
+        pos = (pos > Int64(m_contents.Count())) ? Int64(m_contents.Count()) : pos;
+
+        m_position = UInt(pos);
+    }
+
+    Int64 MemoryStream::Read(void * buffer, Int64 length)
+    {
+        if (!CanRead())
+        {
+            throw IOException("Cannot read this stream.");
+        }
+
+        const Int64 maxRead = Int64(m_contents.Count() - m_position);
+        
+        if (maxRead == 0 && length > 0)
+        {
+            m_atEnd = true;
+            throw EndOfStreamException("End of file is reached.");
+        }
+
+        length = length > maxRead ? maxRead : length;
+
+        ::memcpy(buffer, m_contents.begin() + m_position, size_t(length));
+        m_position += UInt(length);
+        return maxRead;
+    }
+    
+    Int64 MemoryStream::Write(const void * buffer, Int64 length)
+    {
+        if (!CanWrite())
+        {
+            throw IOException("Cannot write this stream.");
+        }
+
+        if (m_position == m_contents.Count())
+        {
+            m_contents.AddRange((const uint8_t*)buffer, UInt(length));
+        }
+        else
+        {
+            m_contents.InsertRange(m_position, (const uint8_t*)buffer, UInt(length));
+        }
+
+        m_atEnd = false;
+
+        m_position += UInt(length);
+        return length;
+    }
+
 }

source/core/stream.h

@@ -53,14 +53,40 @@ namespace Slang
 
 	enum class FileAccess
 	{
-		Read = 1, Write = 2, ReadWrite = 3
+		None = 0, Read = 1, Write = 2, ReadWrite = 3
 	};
 
 	enum class FileShare
 	{
 		None, ReadOnly, WriteOnly, ReadWrite
 	};
 
+    class MemoryStream : public Stream
+    {
+    public:
+        virtual Int64 GetPosition() SLANG_OVERRIDE { return m_position;  }
+        virtual void Seek(SeekOrigin origin, Int64 offset) SLANG_OVERRIDE;
+        virtual Int64 Read(void * buffer, Int64 length) SLANG_OVERRIDE;
+        virtual Int64 Write(const void * buffer, Int64 length) SLANG_OVERRIDE;
+        virtual bool IsEnd() SLANG_OVERRIDE { return m_atEnd; }
+        virtual bool CanRead() SLANG_OVERRIDE { return (int(m_access) & int(FileAccess::Read)) != 0;  }
+        virtual bool CanWrite() SLANG_OVERRIDE { return (int(m_access) & int(FileAccess::Write)) != 0; }
+        virtual void Close() SLANG_OVERRIDE { m_access = FileAccess::None;  }
+
+        MemoryStream(FileAccess access) :
+            m_access(access),
+            m_position(0),
+            m_atEnd(false)
+        {}
+
+        UInt m_position;
+
+        bool m_atEnd;           ///< Happens when a read is done and nothing can be returned because already at end
+
+        FileAccess m_access;
+        List<uint8_t> m_contents;
+    };
+
 	class FileStream : public Stream
 	{
 	private:

source/slang/compiler.cpp

@@ -260,7 +260,7 @@ namespace Slang
             {
                 codeBuilder << "#line 1 \"";
 
-                const String& path = sourceFile->pathInfo.foundPath;
+                const String& path = sourceFile->getPathInfo().foundPath;
 
                 for(auto c : path)
                 {
@@ -277,7 +277,7 @@ namespace Slang
                 }
                 codeBuilder << "\"\n";
 
-                codeBuilder << sourceFile->content << "\n";
+                codeBuilder << sourceFile->getContent() << "\n";
             }
 
             return codeBuilder.ProduceString();
@@ -322,7 +322,7 @@ namespace Slang
                 {
                     codeBuilder << "#line 1 " << translationUnitIndex << "\n";
                 }
-                codeBuilder << sourceFile->content << "\n";
+                codeBuilder << sourceFile->getContent() << "\n";
             }
 
             return codeBuilder.ProduceString();

source/slang/compiler.h

@@ -361,6 +361,9 @@ namespace Slang
         // serialization a bottleneck or firewall between the front end and the backend
         bool useSerialIRBottleneck = false; 
 
+        // If true will serialize and de-serialize with debug information
+        bool verifyDebugSerialization = false;
+
         // How should `#line` directives be emitted (if at all)?
         LineDirectiveMode lineDirectiveMode = LineDirectiveMode::Default;
 

source/slang/diagnostic-defs.h

@@ -446,5 +446,6 @@ DIAGNOSTIC(99999, Internal, internalCompilerError, "Slang internal compiler erro
 DIAGNOSTIC(99999, Error, compilationAborted, "Slang compilation aborted due to internal error");
 DIAGNOSTIC(99999, Error, compilationAbortedDueToException, "Slang compilation aborted due to an exception of $0: $1");
 DIAGNOSTIC(99999, Note, noteLocationOfInternalError, "the Slang compiler threw an exception while working on code near this location");
+DIAGNOSTIC(99999, Internal, serialDebugVerificationFailed, "Verification of serial debug information failed.");
 
 #undef DIAGNOSTIC