Skip to content

Commit 879ec1b

Browse files
jsmall-zzzjsmall-zzz
authored
Feature/source loc refactor (shader-slang#668)
* * Remove the need for IRHighLevelDecoration in Emit * Use the IRLayoutDecoration for GeometryShaderPrimitiveTypeModifier * Initial look at at variable byte encoding, and simple unit test. * Fixing problems with comparison due to naming differences with slang/fxc. * * More tests and perf improvements for byte encoding. * Mechanism to detect processor and processor features in main slang header. * Split out cpu based defines into slang-cpu-defines.h so do not polute slang.h * Support for variable byte encoding on serialization. * Removed unused flag. * Fix warning. * Fix calcMsByte32 for 0 values without using intrinsic. * Fix a mistake in calculating maximum instruction size. * Introduced the idea of SourceUnit. * Small improvements around naming. Add more functionality - including getting the HumaneLoc. * Add support for #line default * Compiling with new SourceLoc handling. * Fix off by one on #line directives. * Can use 32bits for SourceLoc. Fix serialize to use that. * Small fixes and comment on usage. * Premake run. * Fix signed warning. * Fix typo on StringSlicePool::has found in review.
1 parent 60a91d6 commit 879ec1b

18 files changed

+728
-490
lines changed

‎slang.sln

+82-82
Large diffs are not rendered by default.

‎source/core/core.vcxproj

+4-1
Original file line numberDiff line numberDiff line change
@@ -183,11 +183,13 @@
183183
<ClInclude Include="platform.h" />
184184
<ClInclude Include="secure-crt.h" />
185185
<ClInclude Include="slang-byte-encode-util.h" />
186+
<ClInclude Include="slang-cpu-defines.h" />
186187
<ClInclude Include="slang-free-list.h" />
187188
<ClInclude Include="slang-io.h" />
188189
<ClInclude Include="slang-math.h" />
189190
<ClInclude Include="slang-memory-arena.h" />
190191
<ClInclude Include="slang-random-generator.h" />
192+
<ClInclude Include="slang-string-slice-pool.h" />
191193
<ClInclude Include="slang-string-util.h" />
192194
<ClInclude Include="slang-string.h" />
193195
<ClInclude Include="smart-pointer.h" />
@@ -203,14 +205,15 @@
203205
<ClCompile Include="slang-io.cpp" />
204206
<ClCompile Include="slang-memory-arena.cpp" />
205207
<ClCompile Include="slang-random-generator.cpp" />
208+
<ClCompile Include="slang-string-slice-pool.cpp" />
206209
<ClCompile Include="slang-string-util.cpp" />
207210
<ClCompile Include="slang-string.cpp" />
208211
<ClCompile Include="stream.cpp" />
209212
<ClCompile Include="text-io.cpp" />
210213
<ClCompile Include="token-reader.cpp" />
211214
</ItemGroup>
212215
<ItemGroup>
213-
<None Include="core.natvis" />
216+
<Natvis Include="core.natvis" />
214217
</ItemGroup>
215218
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
216219
<ImportGroup Label="ExtensionTargets">

‎source/core/core.vcxproj.filters

+18-9
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<?xml version="1.0" encoding="utf-8"?>
1+
<?xml version="1.0" encoding="utf-8"?>
22
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
33
<ItemGroup>
44
<Filter Include="Header Files">
@@ -45,6 +45,12 @@
4545
<ClInclude Include="secure-crt.h">
4646
<Filter>Header Files</Filter>
4747
</ClInclude>
48+
<ClInclude Include="slang-byte-encode-util.h">
49+
<Filter>Header Files</Filter>
50+
</ClInclude>
51+
<ClInclude Include="slang-cpu-defines.h">
52+
<Filter>Header Files</Filter>
53+
</ClInclude>
4854
<ClInclude Include="slang-free-list.h">
4955
<Filter>Header Files</Filter>
5056
</ClInclude>
@@ -60,6 +66,9 @@
6066
<ClInclude Include="slang-random-generator.h">
6167
<Filter>Header Files</Filter>
6268
</ClInclude>
69+
<ClInclude Include="slang-string-slice-pool.h">
70+
<Filter>Header Files</Filter>
71+
</ClInclude>
6372
<ClInclude Include="slang-string-util.h">
6473
<Filter>Header Files</Filter>
6574
</ClInclude>
@@ -81,14 +90,14 @@
8190
<ClInclude Include="type-traits.h">
8291
<Filter>Header Files</Filter>
8392
</ClInclude>
84-
<ClInclude Include="slang-byte-encode-util.h">
85-
<Filter>Header Files</Filter>
86-
</ClInclude>
8793
</ItemGroup>
8894
<ItemGroup>
8995
<ClCompile Include="platform.cpp">
9096
<Filter>Source Files</Filter>
9197
</ClCompile>
98+
<ClCompile Include="slang-byte-encode-util.cpp">
99+
<Filter>Source Files</Filter>
100+
</ClCompile>
92101
<ClCompile Include="slang-free-list.cpp">
93102
<Filter>Source Files</Filter>
94103
</ClCompile>
@@ -101,6 +110,9 @@
101110
<ClCompile Include="slang-random-generator.cpp">
102111
<Filter>Source Files</Filter>
103112
</ClCompile>
113+
<ClCompile Include="slang-string-slice-pool.cpp">
114+
<Filter>Source Files</Filter>
115+
</ClCompile>
104116
<ClCompile Include="slang-string-util.cpp">
105117
<Filter>Source Files</Filter>
106118
</ClCompile>
@@ -116,13 +128,10 @@
116128
<ClCompile Include="token-reader.cpp">
117129
<Filter>Source Files</Filter>
118130
</ClCompile>
119-
<ClCompile Include="slang-byte-encode-util.cpp">
120-
<Filter>Source Files</Filter>
121-
</ClCompile>
122131
</ItemGroup>
123132
<ItemGroup>
124-
<None Include="core.natvis">
133+
<Natvis Include="core.natvis">
125134
<Filter>Source Files</Filter>
126-
</None>
135+
</Natvis>
127136
</ItemGroup>
128137
</Project>

‎source/core/slang-string-slice-pool.cpp

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#include "slang-string-slice-pool.h"
2+
3+
namespace Slang {
4+
5+
StringSlicePool::StringSlicePool() :
6+
m_arena(1024)
7+
{
8+
clear();
9+
}
10+
11+
void StringSlicePool::clear()
12+
{
13+
m_slices.SetSize(1);
14+
m_slices[0] = UnownedStringSlice::fromLiteral("");
15+
16+
m_map.Clear();
17+
}
18+
19+
StringSlicePool::Handle StringSlicePool::add(const Slice& slice)
20+
{
21+
const int* indexPtr = m_map.TryGetValue(slice);
22+
if (indexPtr)
23+
{
24+
return Handle(*indexPtr);
25+
}
26+
27+
// Create a scoped copy
28+
UnownedStringSlice scopePath(m_arena.allocateString(slice.begin(), slice.size()), slice.size());
29+
30+
const int index = int(m_slices.Count());
31+
32+
m_slices.Add(scopePath);
33+
m_map.Add(scopePath, index);
34+
return Handle(index);
35+
}
36+
37+
int StringSlicePool::findIndex(const Slice& slice) const
38+
{
39+
const int* index = m_map.TryGetValue(slice);
40+
return index ? *index : -1;
41+
42+
}
43+
} // namespace Slang

‎source/core/slang-string-slice-pool.h

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#ifndef SLANG_STRING_SLICE_POOL_H
2+
#define SLANG_STRING_SLICE_POOL_H
3+
4+
#include "slang-string.h"
5+
6+
#include "list.h"
7+
#include "slang-memory-arena.h"
8+
#include "dictionary.h"
9+
10+
namespace Slang {
11+
12+
class StringSlicePool
13+
{
14+
public:
15+
/// Handle of 0 is null. If accessed will be returned as the empty string
16+
enum class Handle : uint32_t;
17+
typedef UnownedStringSlice Slice;
18+
19+
/// Returns the index of a slice, if contained, or -1 if not found
20+
int findIndex(const Slice& slice) const;
21+
22+
/// True if has the slice
23+
bool has(const Slice& slice) { return findIndex(slice) >= 0; }
24+
/// Add a slice
25+
Handle add(const Slice& slice);
26+
27+
/// Empty contents
28+
void clear();
29+
30+
/// Get the slice from the handle
31+
const UnownedStringSlice& getSlice(Handle handle) const { return m_slices[UInt(handle)]; }
32+
33+
/// Get all the slices
34+
const List<UnownedStringSlice>& getSlices() const { return m_slices; }
35+
36+
/// Convert a handle to and index. (A handle is just an index!)
37+
static int asIndex(Handle handle) { return int(handle); }
38+
39+
/// Ctor
40+
StringSlicePool();
41+
42+
protected:
43+
List<UnownedStringSlice> m_slices;
44+
Dictionary<UnownedStringSlice, int> m_map;
45+
MemoryArena m_arena;
46+
};
47+
48+
} // namespace Slang
49+
50+
#endif // SLANG_STRING_SLICE_POOL_H

‎source/slang/diagnostics.cpp

+1-2
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,7 @@ static void formatDiagnostic(
169169
{
170170
auto sourceManager = sink->sourceManager;
171171

172-
auto expandedLoc = sourceManager->expandSourceLoc(diagnostic.loc);
173-
auto humaneLoc = getHumaneLoc(expandedLoc);
172+
auto humaneLoc = sourceManager->getHumaneLoc(diagnostic.loc);
174173

175174
sb << humaneLoc.getPath();
176175
sb << "(";

‎source/slang/ir-serialize.cpp

+5-16
Original file line numberDiff line numberDiff line change
@@ -976,11 +976,8 @@ static size_t _calcInstChunkSize(IRSerialBinary::CompressionType compressionType
976976
SLANG_RETURN_ON_FAIL(_writeArrayChunk(compressionType, Bin::kExternalOperandsFourCc, data.m_externalOperands, stream));
977977
SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, Bin::kStringFourCc, data.m_strings, stream));
978978

979-
{
980-
uint32_t fourCc = sizeof(IRSerialData::RawSourceLoc) == 4 ? Bin::kUInt32SourceLocFourCc : Bin::kUInt64SourceLocFourCc;
981-
SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, fourCc, data.m_rawSourceLocs, stream));
982-
}
983-
979+
SLANG_RETURN_ON_FAIL(_writeArrayChunk(Bin::CompressionType::None, Bin::kUInt32SourceLocFourCc, data.m_rawSourceLocs, stream));
980+
984981
return SLANG_OK;
985982
}
986983

@@ -1337,18 +1334,10 @@ int64_t _calcChunkTotalSize(const IRSerialBinary::Chunk& chunk)
13371334
break;
13381335
}
13391336
case Bin::kUInt32SourceLocFourCc:
1340-
case Bin::kUInt64SourceLocFourCc:
13411337
{
1342-
if ((sizeof(IRSerialData::RawSourceLoc) == 4 && chunk.m_type == Bin::kUInt32SourceLocFourCc) ||
1343-
(sizeof(IRSerialData::RawSourceLoc) == 8 && chunk.m_type == Bin::kUInt64SourceLocFourCc))
1344-
{
1345-
SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_rawSourceLocs));
1346-
remainingBytes -= _calcChunkTotalSize(chunk);
1347-
}
1348-
else
1349-
{
1350-
SLANG_RETURN_ON_FAIL(_skip(chunk, stream, &remainingBytes));
1351-
}
1338+
1339+
SLANG_RETURN_ON_FAIL(_readArrayUncompressedChunk(slangHeader, chunk, stream, &bytesRead, dataOut->m_rawSourceLocs));
1340+
remainingBytes -= _calcChunkTotalSize(chunk);
13521341
break;
13531342
}
13541343
default:

‎source/slang/ir-serialize.h

-2
Original file line numberDiff line numberDiff line change
@@ -256,8 +256,6 @@ struct IRSerialBinary
256256
static const uint32_t kStringFourCc = SLANG_FOUR_CC('S', 'L', 's', 't');
257257
/// 4 bytes per entry
258258
static const uint32_t kUInt32SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '4');
259-
/// 8 bytes per entry
260-
static const uint32_t kUInt64SourceLocFourCc = SLANG_FOUR_CC('S', 'r', 's', '8');
261259

262260
struct SlangHeader
263261
{

‎source/slang/lexer.cpp

+7-23
Original file line numberDiff line numberDiff line change
@@ -84,22 +84,22 @@ namespace Slang
8484
// Lexer
8585

8686
void Lexer::initialize(
87-
SourceFile* inSourceFile,
87+
SourceUnit* inSourceUnit,
8888
DiagnosticSink* inSink,
8989
NamePool* inNamePool)
9090
{
91-
sourceFile = inSourceFile;
91+
sourceUnit = inSourceUnit;
9292
sink = inSink;
9393
namePool = inNamePool;
9494

95-
auto content = inSourceFile->content;
95+
auto content = inSourceUnit->getSourceFile()->content;
9696

9797
begin = content.begin();
9898
cursor = content.begin();
9999
end = content.end();
100100

101-
spellingStartLoc = inSourceFile->sourceRange.begin;
102-
presumedStartLoc = spellingStartLoc;
101+
// Set the start location
102+
startLoc = inSourceUnit->getRange().begin;
103103

104104
tokenFlags = TokenFlag::AtStartOfLine | TokenFlag::AfterWhitespace;
105105
lexerFlags = 0;
@@ -227,7 +227,7 @@ namespace Slang
227227
lexer->tokenFlags |= TokenFlag::ScrubbingNeeded;
228228

229229
// Now try again, looking at the character after the
230-
// escaped nmewline.
230+
// escaped newline.
231231
continue;
232232

233233
default:
@@ -333,23 +333,7 @@ namespace Slang
333333

334334
static SourceLoc getSourceLoc(Lexer* lexer)
335335
{
336-
return lexer->presumedStartLoc + (lexer->cursor - lexer->begin);
337-
}
338-
339-
// Begin overriding the reported locations of tokens,
340-
// based on a `#line` directives
341-
void Lexer::startOverridingSourceLocations(
342-
SourceLoc loc)
343-
{
344-
if(loc.isValid())
345-
{
346-
presumedStartLoc = loc;
347-
}
348-
}
349-
350-
void Lexer::stopOverridingSourceLocations()
351-
{
352-
presumedStartLoc = spellingStartLoc;
336+
return lexer->startLoc + (lexer->cursor - lexer->begin);
353337
}
354338

355339
static void lexDigits(Lexer* lexer, int base)

‎source/slang/lexer.h

+7-12
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ namespace Slang
7373
struct Lexer
7474
{
7575
void initialize(
76-
SourceFile* sourceFile,
76+
SourceUnit* sourceUnit,
7777
DiagnosticSink* sink,
7878
NamePool* namePool);
7979

@@ -83,15 +83,7 @@ namespace Slang
8383

8484
TokenList lexAllTokens();
8585

86-
// Begin overriding the reported locations of tokens,
87-
// based on a `#line` directives
88-
void startOverridingSourceLocations(SourceLoc loc);
89-
90-
// Stop overriding source locations, and go back
91-
// to reporting source locations in the original file
92-
void stopOverridingSourceLocations();
93-
94-
SourceFile* sourceFile;
86+
SourceUnit* sourceUnit;
9587
DiagnosticSink* sink;
9688
NamePool* namePool;
9789

@@ -100,13 +92,16 @@ namespace Slang
10092
char const* begin;
10193
char const* end;
10294

95+
/// The starting sourceLoc (same as first location of SourceUnit)
96+
SourceLoc startLoc;
97+
10398
// The starting source location for the code as written,
10499
// which cannot be overridden.
105-
SourceLoc spellingStartLoc;
100+
//SourceLoc spellingStartLoc;
106101

107102
// The nominal starting location for the file, taking
108103
// any active `#line` directive into account.
109-
SourceLoc presumedStartLoc;
104+
//SourceLoc presumedStartLoc;
110105

111106
TokenFlags tokenFlags;
112107
LexerFlags lexerFlags;

0 commit comments

Comments
 (0)