From 2cdc1cd8ec7c20f897143f3c524c46ea9d644968 Mon Sep 17 00:00:00 2001 From: Adam Cheney Date: Mon, 3 Feb 2025 13:00:09 -0500 Subject: [PATCH 1/7] Fix precompiledTargetModule tests Add SPIRV-Tool linker support to gfx unit tests and use the linker in precompileModule tests that use precompiled modules to reconstitute SPIRV shaders that were modularly compiled. Fix a Slang reference count bug in the precompile service. --- source/slang/slang-compiler-tu.cpp | 1 + tests/expected-failure-github.txt | 3 +- tests/expected-failure.txt | 1 - tools/CMakeLists.txt | 1 + tools/gfx/d3d12/d3d12-shader-program.cpp | 6 +- tools/gfx/d3d12/d3d12-shader-program.h | 2 +- tools/gfx/metal/metal-shader-program.cpp | 8 +- tools/gfx/metal/metal-shader-program.h | 2 +- tools/gfx/renderer-shared.cpp | 114 +++++++++++++++++++---- tools/gfx/renderer-shared.h | 2 +- tools/gfx/slang-context.h | 2 + tools/gfx/vulkan/vk-shader-program.cpp | 52 ++++++++++- tools/gfx/vulkan/vk-shader-program.h | 2 +- 13 files changed, 159 insertions(+), 37 deletions(-) diff --git a/source/slang/slang-compiler-tu.cpp b/source/slang/slang-compiler-tu.cpp index c20fc9a802..8bfe9a9ab7 100644 --- a/source/slang/slang-compiler-tu.cpp +++ b/source/slang/slang-compiler-tu.cpp @@ -291,6 +291,7 @@ SLANG_NO_THROW SlangResult SLANG_MCALL ComponentType::getModuleDependency( { return SLANG_E_INVALID_ARG; } + getModuleDependencies()[dependencyIndex]->addRef(); *outModule = getModuleDependencies()[dependencyIndex]; return SLANG_OK; } diff --git a/tests/expected-failure-github.txt b/tests/expected-failure-github.txt index 60d632785c..38552fa9e6 100644 --- a/tests/expected-failure-github.txt +++ b/tests/expected-failure-github.txt @@ -12,5 +12,4 @@ tests/autodiff/custom-intrinsic.slang.2 syn (wgpu) tests/bugs/buffer-swizzle-store.slang.3 syn (wgpu) tests/compute/interface-shader-param-in-struct.slang.4 syn (wgpu) tests/compute/interface-shader-param.slang.5 syn (wgpu) -tests/language-feature/shader-params/interface-shader-param-ordinary.slang.4 syn (wgpu) -gfx-unit-test-tool/precompiledTargetModule2Vulkan.internal +tests/language-feature/shader-params/interface-shader-param-ordinary.slang.4 syn (wgpu) \ No newline at end of file diff --git a/tests/expected-failure.txt b/tests/expected-failure.txt index 2d95734cc7..7283c8d97c 100644 --- a/tests/expected-failure.txt +++ b/tests/expected-failure.txt @@ -5,4 +5,3 @@ tests/language-feature/saturated-cooperation/fuse.slang (vk) tests/bugs/byte-address-buffer-interlocked-add-f32.slang (vk) tests/ir/loop-unroll-0.slang.1 (vk) tests/hlsl-intrinsic/texture/float-atomics.slang (vk) -gfx-unit-test-tool/precompiledTargetModule2Vulkan.internal diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 9968d7ca8c..c894b100a7 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -141,6 +141,7 @@ if(SLANG_ENABLE_GFX) $<$:X11::X11> $<$:CUDA::cuda_driver> $<$:${NVAPI_LIBRARIES}> + SPIRV-Tools-link LINK_WITH_FRAMEWORK Foundation Cocoa QuartzCore Metal EXTRA_COMPILE_DEFINITIONS_PRIVATE $<$:GFX_ENABLE_CUDA> diff --git a/tools/gfx/d3d12/d3d12-shader-program.cpp b/tools/gfx/d3d12/d3d12-shader-program.cpp index a39f204651..c5fcf75809 100644 --- a/tools/gfx/d3d12/d3d12-shader-program.cpp +++ b/tools/gfx/d3d12/d3d12-shader-program.cpp @@ -10,14 +10,14 @@ using namespace Slang; Result ShaderProgramImpl::createShaderModule( slang::EntryPointReflection* entryPointInfo, - ComPtr kernelCode) + List> kernelCodes) { ShaderBinary shaderBin; shaderBin.stage = entryPointInfo->getStage(); shaderBin.entryPointInfo = entryPointInfo; shaderBin.code.addRange( - reinterpret_cast(kernelCode->getBufferPointer()), - (Index)kernelCode->getBufferSize()); + reinterpret_cast(kernelCodes[0]->getBufferPointer()), + (Index)kernelCodes[0]->getBufferSize()); m_shaders.add(_Move(shaderBin)); return SLANG_OK; } diff --git a/tools/gfx/d3d12/d3d12-shader-program.h b/tools/gfx/d3d12/d3d12-shader-program.h index 669bce960f..fad4e7ad01 100644 --- a/tools/gfx/d3d12/d3d12-shader-program.h +++ b/tools/gfx/d3d12/d3d12-shader-program.h @@ -27,7 +27,7 @@ class ShaderProgramImpl : public ShaderProgramBase virtual Result createShaderModule( slang::EntryPointReflection* entryPointInfo, - ComPtr kernelCode) override; + List> kernelCodes) override; }; } // namespace d3d12 diff --git a/tools/gfx/metal/metal-shader-program.cpp b/tools/gfx/metal/metal-shader-program.cpp index c95e99d047..0bb20740b8 100644 --- a/tools/gfx/metal/metal-shader-program.cpp +++ b/tools/gfx/metal/metal-shader-program.cpp @@ -21,16 +21,16 @@ ShaderProgramImpl::~ShaderProgramImpl() {} Result ShaderProgramImpl::createShaderModule( slang::EntryPointReflection* entryPointInfo, - ComPtr kernelCode) + List> kernelCodes) { Module module; module.stage = entryPointInfo->getStage(); module.entryPointName = entryPointInfo->getNameOverride(); - module.code = kernelCode; + module.code = kernelCodes[0]; dispatch_data_t data = dispatch_data_create( - kernelCode->getBufferPointer(), - kernelCode->getBufferSize(), + kernelCodes[0]->getBufferPointer(), + kernelCodes[0]->getBufferSize(), dispatch_get_main_queue(), NULL); NS::Error* error; diff --git a/tools/gfx/metal/metal-shader-program.h b/tools/gfx/metal/metal-shader-program.h index 691e12c9e4..d5b72d696b 100644 --- a/tools/gfx/metal/metal-shader-program.h +++ b/tools/gfx/metal/metal-shader-program.h @@ -33,7 +33,7 @@ class ShaderProgramImpl : public ShaderProgramBase virtual Result createShaderModule( slang::EntryPointReflection* entryPointInfo, - ComPtr kernelCode) override; + List> kernelCodes) override; }; diff --git a/tools/gfx/renderer-shared.cpp b/tools/gfx/renderer-shared.cpp index 9f5574dccf..50d823c1e3 100644 --- a/tools/gfx/renderer-shared.cpp +++ b/tools/gfx/renderer-shared.cpp @@ -1103,32 +1103,106 @@ void ShaderProgramBase::init(const IShaderProgram::Desc& inDesc) Result ShaderProgramBase::compileShaders(RendererBase* device) { + auto compileTarget = device->slangContext.compileTarget; // For a fully specialized program, read and store its kernel code in `shaderProgram`. auto compileShader = [&](slang::EntryPointReflection* entryPointInfo, slang::IComponentType* entryPointComponent, SlangInt entryPointIndex) { auto stage = entryPointInfo->getStage(); - ComPtr kernelCode; - ComPtr diagnostics; - auto compileResult = device->getEntryPointCodeFromShaderCache( - entryPointComponent, - entryPointIndex, - 0, - kernelCode.writeRef(), - diagnostics.writeRef()); - if (diagnostics) + List> kernelCodes; { - DebugMessageType msgType = DebugMessageType::Warning; - if (compileResult != SLANG_OK) - msgType = DebugMessageType::Error; - getDebugCallback()->handleMessage( - msgType, - DebugMessageSource::Slang, - (char*)diagnostics->getBufferPointer()); + ComPtr downstreamIR; + ComPtr diagnostics; + auto compileResult = device->getEntryPointCodeFromShaderCache( + entryPointComponent, + entryPointIndex, + 0, + downstreamIR.writeRef(), + diagnostics.writeRef()); + if (diagnostics) + { + DebugMessageType msgType = DebugMessageType::Warning; + if (compileResult != SLANG_OK) + msgType = DebugMessageType::Error; + getDebugCallback()->handleMessage( + msgType, + DebugMessageSource::Slang, + (char*)diagnostics->getBufferPointer()); + } + kernelCodes.add(downstreamIR); } - SLANG_RETURN_ON_FAIL(compileResult); - SLANG_RETURN_ON_FAIL(createShaderModule(entryPointInfo, kernelCode)); + + // If target precompilation was used, kernelCode may only represent the + // glue code holding together the bits of precompiled target IR. + // Collect those dependency target IRs too. + ComPtr componentPrecompileService; + if (entryPointComponent->queryInterface( + slang::IModulePrecompileService_Experimental::getTypeGuid(), + (void**)componentPrecompileService.writeRef()) == SLANG_OK) + { + SlangInt dependencyCount = componentPrecompileService->getModuleDependencyCount(); + if (dependencyCount > 0) + { + for (int dependencyIndex = 0; dependencyIndex < dependencyCount; dependencyIndex++) + { + ComPtr dependencyModule; + { + ComPtr diagnosticsBlob; + auto result = componentPrecompileService->getModuleDependency( + dependencyIndex, + dependencyModule.writeRef(), + diagnosticsBlob.writeRef()); + if (diagnosticsBlob) + { + DebugMessageType msgType = DebugMessageType::Warning; + if (result != SLANG_OK) + msgType = DebugMessageType::Error; + getDebugCallback()->handleMessage( + msgType, + DebugMessageSource::Slang, + (char*)diagnosticsBlob->getBufferPointer()); + } + SLANG_RETURN_ON_FAIL(result); + } + + ComPtr downstreamIR; + { + ComPtr diagnosticsBlob; + SlangResult result = SLANG_OK; + ComPtr precompileService; + result = dependencyModule->queryInterface( + slang::IModulePrecompileService_Experimental::getTypeGuid(), + (void**)precompileService.writeRef()); + if (result == SLANG_OK) + { + ComPtr diagnosticsBlob; + auto result = precompileService->getPrecompiledTargetCode( + compileTarget, + downstreamIR.writeRef(), + diagnosticsBlob.writeRef()); + if (result == SLANG_OK) + { + kernelCodes.add(downstreamIR); + } + if (diagnosticsBlob) + { + DebugMessageType msgType = DebugMessageType::Warning; + if (result != SLANG_OK) + msgType = DebugMessageType::Error; + getDebugCallback()->handleMessage( + msgType, + DebugMessageSource::Slang, + (char*)diagnosticsBlob->getBufferPointer()); + } + } + SLANG_RETURN_ON_FAIL(result); + } + } + } + } + + SLANG_RETURN_ON_FAIL(createShaderModule(entryPointInfo, kernelCodes)); return SLANG_OK; }; @@ -1160,10 +1234,10 @@ Result ShaderProgramBase::compileShaders(RendererBase* device) Result ShaderProgramBase::createShaderModule( slang::EntryPointReflection* entryPointInfo, - ComPtr kernelCode) + List> kernelCodes) { SLANG_UNUSED(entryPointInfo); - SLANG_UNUSED(kernelCode); + SLANG_UNUSED(kernelCodes); return SLANG_OK; } diff --git a/tools/gfx/renderer-shared.h b/tools/gfx/renderer-shared.h index c763c9ba28..17d3d3fc0e 100644 --- a/tools/gfx/renderer-shared.h +++ b/tools/gfx/renderer-shared.h @@ -877,7 +877,7 @@ class ShaderProgramBase : public IShaderProgram, public Slang::ComObject Slang::Result compileShaders(RendererBase* device); virtual Slang::Result createShaderModule( slang::EntryPointReflection* entryPointInfo, - Slang::ComPtr kernelCode); + Slang::List> kernelCodes); virtual SLANG_NO_THROW slang::TypeReflection* SLANG_MCALL findTypeByName(const char* name) override diff --git a/tools/gfx/slang-context.h b/tools/gfx/slang-context.h index 719c70b506..5494ccfcea 100644 --- a/tools/gfx/slang-context.h +++ b/tools/gfx/slang-context.h @@ -10,6 +10,7 @@ class SlangContext public: Slang::ComPtr globalSession; Slang::ComPtr session; + SlangCompileTarget compileTarget; Result initialize( const gfx::IDevice::SlangDesc& desc, uint32_t extendedDescCount, @@ -27,6 +28,7 @@ class SlangContext SLANG_RETURN_ON_FAIL(slang::createGlobalSession(globalSession.writeRef())); } + this->compileTarget = compileTarget; slang::SessionDesc slangSessionDesc = {}; slangSessionDesc.defaultMatrixLayoutMode = desc.defaultMatrixLayoutMode; slangSessionDesc.searchPathCount = desc.searchPathCount; diff --git a/tools/gfx/vulkan/vk-shader-program.cpp b/tools/gfx/vulkan/vk-shader-program.cpp index 43a2957866..947a93ff7a 100644 --- a/tools/gfx/vulkan/vk-shader-program.cpp +++ b/tools/gfx/vulkan/vk-shader-program.cpp @@ -4,6 +4,8 @@ #include "vk-device.h" #include "vk-util.h" +#include "external/spirv-tools/include/spirv-tools/linker.hpp" + namespace gfx { @@ -69,17 +71,61 @@ VkPipelineShaderStageCreateInfo ShaderProgramImpl::compileEntryPoint( return shaderStageCreateInfo; } +static ComPtr LinkUsingSPIRVTools(List > kernelCodes) +{ + spvtools::Context context(SPV_ENV_UNIVERSAL_1_5); + spvtools::LinkerOptions options; + spvtools::MessageConsumer consumer = [](spv_message_level_t level, + const char* source, + const spv_position_t& position, + const char* message) + { + printf("SPIRV-TOOLS: %s\n", message); + printf("SPIRV-TOOLS: %s\n", source); + printf("SPIRV-TOOLS: %zu:%zu\n", position.index, position.column); + }; + context.SetMessageConsumer(consumer); + std::vector binaries; + std::vector binary_sizes; + for (auto kernelCode : kernelCodes) + { + binaries.push_back((uint32_t*)kernelCode->getBufferPointer()); + binary_sizes.push_back(kernelCode->getBufferSize() / sizeof(uint32_t)); + } + + std::vector linked_binary; + + spvtools::Link( + context, + binaries.data(), + binary_sizes.data(), + binaries.size(), + &linked_binary, + options); + + // Create a blob to hold the linked binary + ComPtr linkedKernelCode; + + // Replace kernel code with linked binary + // Creates a new blob with the linked binary + linkedKernelCode = RawBlob::create(linked_binary.data(), linked_binary.size() * sizeof(uint32_t)); + + return linkedKernelCode; +} + Result ShaderProgramImpl::createShaderModule( slang::EntryPointReflection* entryPointInfo, - ComPtr kernelCode) + List> kernelCodes) { - m_codeBlobs.add(kernelCode); + ComPtr linkedKernel = LinkUsingSPIRVTools(kernelCodes); + m_codeBlobs.add(linkedKernel); + VkShaderModule shaderModule; auto realEntryPointName = entryPointInfo->getNameOverride(); const char* spirvBinaryEntryPointName = "main"; m_stageCreateInfos.add(compileEntryPoint( spirvBinaryEntryPointName, - kernelCode, + linkedKernel, (VkShaderStageFlagBits)VulkanUtil::getShaderStage(entryPointInfo->getStage()), shaderModule)); m_entryPointNames.add(realEntryPointName); diff --git a/tools/gfx/vulkan/vk-shader-program.h b/tools/gfx/vulkan/vk-shader-program.h index 3fd56669ae..0fd01eb4cf 100644 --- a/tools/gfx/vulkan/vk-shader-program.h +++ b/tools/gfx/vulkan/vk-shader-program.h @@ -37,7 +37,7 @@ class ShaderProgramImpl : public ShaderProgramBase virtual Result createShaderModule( slang::EntryPointReflection* entryPointInfo, - ComPtr kernelCode) override; + List> kernelCodes) override; }; From 8ab351e10994b911c0207b59d5de2b39cf52dfd8 Mon Sep 17 00:00:00 2001 From: acheney Date: Tue, 25 Feb 2025 20:58:01 -0500 Subject: [PATCH 2/7] Use sm_6_6 New DXC requires higher version for linkability. --- tools/gfx-unit-test/precompiled-module-2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gfx-unit-test/precompiled-module-2.cpp b/tools/gfx-unit-test/precompiled-module-2.cpp index f5d576a232..ca9f8b565a 100644 --- a/tools/gfx-unit-test/precompiled-module-2.cpp +++ b/tools/gfx-unit-test/precompiled-module-2.cpp @@ -112,7 +112,7 @@ void precompiledModule2TestImplCommon( { case gfx::DeviceType::DirectX12: targetDesc.format = SLANG_DXIL; - targetDesc.profile = device->getSlangSession()->getGlobalSession()->findProfile("sm_6_1"); + targetDesc.profile = device->getSlangSession()->getGlobalSession()->findProfile("sm_6_6"); break; case gfx::DeviceType::Vulkan: targetDesc.format = SLANG_SPIRV; From e3f519beb33f3b6e08ac42d029a1c9229f7f03e0 Mon Sep 17 00:00:00 2001 From: acheney Date: Wed, 26 Feb 2025 09:11:41 -0500 Subject: [PATCH 3/7] Rename helper function, pass by reference --- tools/gfx/d3d12/d3d12-shader-program.cpp | 2 +- tools/gfx/d3d12/d3d12-shader-program.h | 2 +- tools/gfx/renderer-shared.cpp | 2 +- tools/gfx/renderer-shared.h | 2 +- tools/gfx/vulkan/vk-shader-program.cpp | 6 +++--- tools/gfx/vulkan/vk-shader-program.h | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/gfx/d3d12/d3d12-shader-program.cpp b/tools/gfx/d3d12/d3d12-shader-program.cpp index c5fcf75809..e52fa320e2 100644 --- a/tools/gfx/d3d12/d3d12-shader-program.cpp +++ b/tools/gfx/d3d12/d3d12-shader-program.cpp @@ -10,7 +10,7 @@ using namespace Slang; Result ShaderProgramImpl::createShaderModule( slang::EntryPointReflection* entryPointInfo, - List> kernelCodes) + List>& kernelCodes) { ShaderBinary shaderBin; shaderBin.stage = entryPointInfo->getStage(); diff --git a/tools/gfx/d3d12/d3d12-shader-program.h b/tools/gfx/d3d12/d3d12-shader-program.h index fad4e7ad01..b95eb030c5 100644 --- a/tools/gfx/d3d12/d3d12-shader-program.h +++ b/tools/gfx/d3d12/d3d12-shader-program.h @@ -27,7 +27,7 @@ class ShaderProgramImpl : public ShaderProgramBase virtual Result createShaderModule( slang::EntryPointReflection* entryPointInfo, - List> kernelCodes) override; + List>& kernelCodes) override; }; } // namespace d3d12 diff --git a/tools/gfx/renderer-shared.cpp b/tools/gfx/renderer-shared.cpp index 50d823c1e3..5349049016 100644 --- a/tools/gfx/renderer-shared.cpp +++ b/tools/gfx/renderer-shared.cpp @@ -1234,7 +1234,7 @@ Result ShaderProgramBase::compileShaders(RendererBase* device) Result ShaderProgramBase::createShaderModule( slang::EntryPointReflection* entryPointInfo, - List> kernelCodes) + List>& kernelCodes) { SLANG_UNUSED(entryPointInfo); SLANG_UNUSED(kernelCodes); diff --git a/tools/gfx/renderer-shared.h b/tools/gfx/renderer-shared.h index 17d3d3fc0e..18270c6cd9 100644 --- a/tools/gfx/renderer-shared.h +++ b/tools/gfx/renderer-shared.h @@ -877,7 +877,7 @@ class ShaderProgramBase : public IShaderProgram, public Slang::ComObject Slang::Result compileShaders(RendererBase* device); virtual Slang::Result createShaderModule( slang::EntryPointReflection* entryPointInfo, - Slang::List> kernelCodes); + Slang::List>& kernelCodes); virtual SLANG_NO_THROW slang::TypeReflection* SLANG_MCALL findTypeByName(const char* name) override diff --git a/tools/gfx/vulkan/vk-shader-program.cpp b/tools/gfx/vulkan/vk-shader-program.cpp index 947a93ff7a..9da279d70f 100644 --- a/tools/gfx/vulkan/vk-shader-program.cpp +++ b/tools/gfx/vulkan/vk-shader-program.cpp @@ -71,7 +71,7 @@ VkPipelineShaderStageCreateInfo ShaderProgramImpl::compileEntryPoint( return shaderStageCreateInfo; } -static ComPtr LinkUsingSPIRVTools(List > kernelCodes) +static ComPtr linkUsingSPIRVTools(List > kernelCodes) { spvtools::Context context(SPV_ENV_UNIVERSAL_1_5); spvtools::LinkerOptions options; @@ -115,9 +115,9 @@ static ComPtr LinkUsingSPIRVTools(List > kernelCo Result ShaderProgramImpl::createShaderModule( slang::EntryPointReflection* entryPointInfo, - List> kernelCodes) + List>& kernelCodes) { - ComPtr linkedKernel = LinkUsingSPIRVTools(kernelCodes); + ComPtr linkedKernel = linkUsingSPIRVTools(kernelCodes); m_codeBlobs.add(linkedKernel); VkShaderModule shaderModule; diff --git a/tools/gfx/vulkan/vk-shader-program.h b/tools/gfx/vulkan/vk-shader-program.h index 0fd01eb4cf..56b3546124 100644 --- a/tools/gfx/vulkan/vk-shader-program.h +++ b/tools/gfx/vulkan/vk-shader-program.h @@ -37,7 +37,7 @@ class ShaderProgramImpl : public ShaderProgramBase virtual Result createShaderModule( slang::EntryPointReflection* entryPointInfo, - List> kernelCodes) override; + List>& kernelCodes) override; }; From 0bfc2ccd63fecab1416863e460b38801e886dc98 Mon Sep 17 00:00:00 2001 From: acheney Date: Wed, 26 Feb 2025 13:26:52 -0500 Subject: [PATCH 4/7] Link through slang-glslang --- source/slang-glslang/CMakeLists.txt | 2 +- source/slang-glslang/slang-glslang.cpp | 63 ++++++++++++++++++++++++++ source/slang-glslang/slang-glslang.h | 11 ++++- tools/CMakeLists.txt | 1 - tools/gfx/vulkan/vk-device.cpp | 1 + tools/gfx/vulkan/vk-device.h | 2 + tools/gfx/vulkan/vk-shader-program.cpp | 49 +++----------------- 7 files changed, 83 insertions(+), 46 deletions(-) diff --git a/source/slang-glslang/CMakeLists.txt b/source/slang-glslang/CMakeLists.txt index c457ad4722..e1ced542db 100644 --- a/source/slang-glslang/CMakeLists.txt +++ b/source/slang-glslang/CMakeLists.txt @@ -6,7 +6,7 @@ if(SLANG_ENABLE_SLANG_GLSLANG) . MODULE USE_FEWER_WARNINGS - LINK_WITH_PRIVATE glslang SPIRV SPIRV-Tools-opt + LINK_WITH_PRIVATE glslang SPIRV SPIRV-Tools-opt SPIRV-Tools-link INCLUDE_DIRECTORIES_PRIVATE ${slang_SOURCE_DIR}/include INSTALL EXPORT_SET_NAME SlangTargets diff --git a/source/slang-glslang/slang-glslang.cpp b/source/slang-glslang/slang-glslang.cpp index ca45be05b0..419ae6d73f 100644 --- a/source/slang-glslang/slang-glslang.cpp +++ b/source/slang-glslang/slang-glslang.cpp @@ -7,6 +7,7 @@ #include "slang.h" #include "spirv-tools/libspirv.h" #include "spirv-tools/optimizer.hpp" +#include "spirv-tools/linker.hpp" #ifdef _WIN32 #include @@ -977,3 +978,65 @@ extern "C" request.set(*inRequest); return glslang_compile_1_1(&request); } + +extern "C" +#ifdef _MSC_VER + _declspec(dllexport) +#else + __attribute__((__visibility__("default"))) +#endif + int glslang_linkSPIRV(glslang_LinkRequest* request) +{ + if (!request || !request->modules || request->linkResult) + return false; + + try + { + spvtools::Context context(SPV_ENV_UNIVERSAL_1_5); + spvtools::LinkerOptions options = {}; + + spvtools::MessageConsumer consumer = [](spv_message_level_t level, + const char* source, + const spv_position_t& position, + const char* message) + { + printf("SPIRV-TOOLS: %s\n", message); + printf("SPIRV-TOOLS: %s\n", source); + printf("SPIRV-TOOLS: %zu:%zu\n", position.index, position.column); + }; + context.SetMessageConsumer(consumer); + + std::vector> moduleVecs(request->moduleCount); + std::vector moduleData(request->moduleCount); + std::vector moduleSizes(request->moduleCount); + + for (size_t i = 0; i < request->moduleCount; ++i) + { + moduleData[i] = request->modules[i]; + moduleSizes[i] = request->moduleSizes[i]; + } + + std::vector linkedBinary; + spv_result_t success = spvtools::Link( + context, + moduleData.data(), + moduleSizes.data(), + request->moduleCount, + &linkedBinary, + options); + + if (success == SPV_SUCCESS) + { + request->linkResult = new uint32_t[linkedBinary.size()]; + memcpy((void*)request->linkResult, linkedBinary.data(), + linkedBinary.size() * sizeof(uint32_t)); + request->linkResultSize = linkedBinary.size(); + } + + return success; + } + catch(...) + { + return false; + } +} diff --git a/source/slang-glslang/slang-glslang.h b/source/slang-glslang/slang-glslang.h index 7b955c5af7..8343b4c8e1 100644 --- a/source/slang-glslang/slang-glslang.h +++ b/source/slang-glslang/slang-glslang.h @@ -152,10 +152,19 @@ inline void glslang_CompileRequest_1_2::set(const glslang_CompileRequest_1_1& in memcpy(this, &in, sizeof(in)); } +typedef struct glslang_LinkRequest_t +{ + const uint32_t** modules; // Input: array of pointers to SPIR-V modules + const uint32_t* moduleSizes; // Input: array of sizes of SPIR-V modules in 32-bit words + int moduleCount; // Input: number of modules in the array + const uint32_t* linkResult; // Output: pointer to linked SPIR-V module + size_t linkResultSize; // Output: size of the linked SPIR-V module in 32-bit words +} glslang_LinkRequest; + typedef int (*glslang_CompileFunc_1_0)(glslang_CompileRequest_1_0* request); typedef int (*glslang_CompileFunc_1_1)(glslang_CompileRequest_1_1* request); typedef int (*glslang_CompileFunc_1_2)(glslang_CompileRequest_1_2* request); typedef bool (*glslang_ValidateSPIRVFunc)(const uint32_t* contents, int contentsSize); typedef bool (*glslang_DisassembleSPIRVFunc)(const uint32_t* contents, int contentsSize); - +typedef bool (*glslang_LinkSPIRVFunc)(glslang_LinkRequest* request); #endif diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index c894b100a7..9968d7ca8c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -141,7 +141,6 @@ if(SLANG_ENABLE_GFX) $<$:X11::X11> $<$:CUDA::cuda_driver> $<$:${NVAPI_LIBRARIES}> - SPIRV-Tools-link LINK_WITH_FRAMEWORK Foundation Cocoa QuartzCore Metal EXTRA_COMPILE_DEFINITIONS_PRIVATE $<$:GFX_ENABLE_CUDA> diff --git a/tools/gfx/vulkan/vk-device.cpp b/tools/gfx/vulkan/vk-device.cpp index cffa094ae7..981b5abacc 100644 --- a/tools/gfx/vulkan/vk-device.cpp +++ b/tools/gfx/vulkan/vk-device.cpp @@ -1019,6 +1019,7 @@ SlangResult DeviceImpl::initialize(const Desc& desc) SLANG_RETURN_ON_FAIL(RendererBase::initialize(desc)); SlangResult initDeviceResult = SLANG_OK; + m_glslang.init(); for (int forceSoftware = 0; forceSoftware <= 1; forceSoftware++) { initDeviceResult = m_module.init(forceSoftware != 0); diff --git a/tools/gfx/vulkan/vk-device.h b/tools/gfx/vulkan/vk-device.h index 3b6c83103d..dc37137122 100644 --- a/tools/gfx/vulkan/vk-device.h +++ b/tools/gfx/vulkan/vk-device.h @@ -3,6 +3,7 @@ #include "vk-base.h" #include "vk-framebuffer.h" +#include "glslang-module.h" namespace gfx { @@ -196,6 +197,7 @@ class DeviceImpl : public RendererBase VulkanModule m_module; VulkanApi m_api; + GlslangModule m_glslang; VulkanDeviceQueue m_deviceQueue; uint32_t m_queueFamilyIndex; diff --git a/tools/gfx/vulkan/vk-shader-program.cpp b/tools/gfx/vulkan/vk-shader-program.cpp index 9da279d70f..7fe18c3af3 100644 --- a/tools/gfx/vulkan/vk-shader-program.cpp +++ b/tools/gfx/vulkan/vk-shader-program.cpp @@ -71,53 +71,16 @@ VkPipelineShaderStageCreateInfo ShaderProgramImpl::compileEntryPoint( return shaderStageCreateInfo; } -static ComPtr linkUsingSPIRVTools(List > kernelCodes) -{ - spvtools::Context context(SPV_ENV_UNIVERSAL_1_5); - spvtools::LinkerOptions options; - spvtools::MessageConsumer consumer = [](spv_message_level_t level, - const char* source, - const spv_position_t& position, - const char* message) - { - printf("SPIRV-TOOLS: %s\n", message); - printf("SPIRV-TOOLS: %s\n", source); - printf("SPIRV-TOOLS: %zu:%zu\n", position.index, position.column); - }; - context.SetMessageConsumer(consumer); - std::vector binaries; - std::vector binary_sizes; - for (auto kernelCode : kernelCodes) - { - binaries.push_back((uint32_t*)kernelCode->getBufferPointer()); - binary_sizes.push_back(kernelCode->getBufferSize() / sizeof(uint32_t)); - } - - std::vector linked_binary; - - spvtools::Link( - context, - binaries.data(), - binary_sizes.data(), - binaries.size(), - &linked_binary, - options); - - // Create a blob to hold the linked binary - ComPtr linkedKernelCode; - - // Replace kernel code with linked binary - // Creates a new blob with the linked binary - linkedKernelCode = RawBlob::create(linked_binary.data(), linked_binary.size() * sizeof(uint32_t)); - - return linkedKernelCode; -} - Result ShaderProgramImpl::createShaderModule( slang::EntryPointReflection* entryPointInfo, List>& kernelCodes) { - ComPtr linkedKernel = linkUsingSPIRVTools(kernelCodes); + ComPtr linkedKernel = m_device->m_glslang.linkSPIRV(kernelCodes); + if (!linkedKernel) + { + return SLANG_FAIL; + } + m_codeBlobs.add(linkedKernel); VkShaderModule shaderModule; From e3b2bc721e2944fb1953c25d3556cb58626703ed Mon Sep 17 00:00:00 2001 From: acheney Date: Wed, 26 Feb 2025 13:42:54 -0500 Subject: [PATCH 5/7] Add missing files --- tools/gfx/vulkan/glslang-module.cpp | 107 ++++++++++++++++++++++++++++ tools/gfx/vulkan/glslang-module.h | 36 ++++++++++ 2 files changed, 143 insertions(+) create mode 100644 tools/gfx/vulkan/glslang-module.cpp create mode 100644 tools/gfx/vulkan/glslang-module.h diff --git a/tools/gfx/vulkan/glslang-module.cpp b/tools/gfx/vulkan/glslang-module.cpp new file mode 100644 index 0000000000..c2817cfce9 --- /dev/null +++ b/tools/gfx/vulkan/glslang-module.cpp @@ -0,0 +1,107 @@ +// glslang-module.cpp +#include "glslang-module.h" + +#include +#include +#include + +#if SLANG_WINDOWS_FAMILY +#include +#else +#include +#endif + +#include "../renderer-shared.h" + +namespace gfx +{ +using namespace Slang; + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! GlslangModule !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +Slang::Result GlslangModule::init() +{ + if (isInitialized()) + { + destroy(); + } + + const char* dynamicLibraryName = "Unknown"; + +#if SLANG_WINDOWS_FAMILY + dynamicLibraryName = "slang-glslang.dll"; + HMODULE module = ::LoadLibraryA(dynamicLibraryName); + m_module = (void*)module; +#elif SLANG_APPLE_FAMILY + dynamicLibraryName = "libslang_glslang.dylib"; + m_module = dlopen(dynamicLibraryName, RTLD_NOW | RTLD_GLOBAL); +#else + dynamicLibraryName = "libslang_glslang.so"; + m_module = dlopen(dynamicLibraryName, RTLD_NOW); +#endif + + if (!m_module) + { + return SLANG_FAIL; + } + + // Load functions +#if SLANG_WINDOWS_FAMILY + m_linkSPIRVFunc = (glslang_LinkSPIRVFunc)GetProcAddress((HMODULE)m_module, "glslang_linkSPIRV"); +#else + m_linkSPIRVFunc = (glslang_LinkSPIRVFunc)dlsym(m_module, "glslang_linkSPIRV"); +#endif + if (!m_linkSPIRVFunc) + { + return SLANG_FAIL; + } + + return SLANG_OK; +} + +void GlslangModule::destroy() +{ + if (!isInitialized()) + { + return; + } + +#if SLANG_WINDOWS_FAMILY + ::FreeLibrary((HMODULE)m_module); +#else + dlclose(m_module); +#endif + m_module = nullptr; +} + +ComPtr GlslangModule::linkSPIRV(List> spirvModules) +{ + + if (!m_linkSPIRVFunc) + { + return nullptr; + } + + glslang_LinkRequest request = {}; + + std::vector moduleCodePtrs(spirvModules.getCount()); + std::vector moduleSizes(spirvModules.getCount()); + for (Index i = 0; i < spirvModules.getCount(); ++i) + { + moduleCodePtrs[i] = (const uint32_t*)spirvModules[i]->getBufferPointer(); + moduleSizes[i] = spirvModules[i]->getBufferSize() / sizeof(uint32_t); + SLANG_ASSERT(spirvModules[i]->getBufferSize() % sizeof(uint32_t) == 0); + } + request.modules = moduleCodePtrs.data(); + request.moduleSizes = moduleSizes.data(); + request.moduleCount = spirvModules.getCount(); + request.linkResult = nullptr; + + m_linkSPIRVFunc(&request); + + ComPtr linkedSPIRV; + linkedSPIRV = RawBlob::create(request.linkResult, request.linkResultSize * sizeof(uint32_t)); + return linkedSPIRV; +} + +} // namespace gfx diff --git a/tools/gfx/vulkan/glslang-module.h b/tools/gfx/vulkan/glslang-module.h new file mode 100644 index 0000000000..fedf7f33c9 --- /dev/null +++ b/tools/gfx/vulkan/glslang-module.h @@ -0,0 +1,36 @@ +// glslang-module.h +#pragma once + +#include "slang-com-ptr.h" +#include "slang-com-helper.h" +#include "core/slang-list.h" +#include "slang.h" +#include "slang-glslang/slang-glslang.h" +#include "external/spirv-tools/include/spirv-tools/linker.hpp" + +namespace gfx +{ + +struct GlslangModule +{ + /// true if has been initialized + SLANG_FORCE_INLINE bool isInitialized() const { return m_module != nullptr; } + + /// Initialize + Slang::Result init(); + + /// Destroy + void destroy(); + + /// Dtor + ~GlslangModule() { destroy(); } + + Slang::ComPtr linkSPIRV(Slang::List> spirvModules); + +protected: + void* m_module = nullptr; + + glslang_LinkSPIRVFunc m_linkSPIRVFunc = nullptr; +}; + +} // namespace gfx From a796e6beb7e18e95e2768b2c9085282b6bc47381 Mon Sep 17 00:00:00 2001 From: acheney Date: Wed, 26 Feb 2025 13:55:05 -0500 Subject: [PATCH 6/7] Fix metal --- tools/gfx/metal/metal-shader-program.cpp | 2 +- tools/gfx/metal/metal-shader-program.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/gfx/metal/metal-shader-program.cpp b/tools/gfx/metal/metal-shader-program.cpp index 0bb20740b8..3e7818454b 100644 --- a/tools/gfx/metal/metal-shader-program.cpp +++ b/tools/gfx/metal/metal-shader-program.cpp @@ -21,7 +21,7 @@ ShaderProgramImpl::~ShaderProgramImpl() {} Result ShaderProgramImpl::createShaderModule( slang::EntryPointReflection* entryPointInfo, - List> kernelCodes) + List>& kernelCodes) { Module module; module.stage = entryPointInfo->getStage(); diff --git a/tools/gfx/metal/metal-shader-program.h b/tools/gfx/metal/metal-shader-program.h index d5b72d696b..c92e432be8 100644 --- a/tools/gfx/metal/metal-shader-program.h +++ b/tools/gfx/metal/metal-shader-program.h @@ -33,7 +33,7 @@ class ShaderProgramImpl : public ShaderProgramBase virtual Result createShaderModule( slang::EntryPointReflection* entryPointInfo, - List> kernelCodes) override; + List>& kernelCodes) override; }; From 5dff8a4cf1bd09be74fc45a1975bb836afac1956 Mon Sep 17 00:00:00 2001 From: slangbot <186143334+slangbot@users.noreply.github.com> Date: Wed, 26 Feb 2025 20:41:46 +0000 Subject: [PATCH 7/7] format code --- source/slang-glslang/slang-glslang.cpp | 8 +++++--- tools/gfx/renderer-shared.cpp | 4 ++-- tools/gfx/vulkan/glslang-module.cpp | 3 ++- tools/gfx/vulkan/glslang-module.h | 8 ++++---- tools/gfx/vulkan/vk-device.h | 2 +- tools/gfx/vulkan/vk-shader-program.cpp | 3 +-- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/source/slang-glslang/slang-glslang.cpp b/source/slang-glslang/slang-glslang.cpp index 419ae6d73f..82cb22e574 100644 --- a/source/slang-glslang/slang-glslang.cpp +++ b/source/slang-glslang/slang-glslang.cpp @@ -6,8 +6,8 @@ #include "glslang/Public/ShaderLang.h" #include "slang.h" #include "spirv-tools/libspirv.h" -#include "spirv-tools/optimizer.hpp" #include "spirv-tools/linker.hpp" +#include "spirv-tools/optimizer.hpp" #ifdef _WIN32 #include @@ -1028,14 +1028,16 @@ extern "C" if (success == SPV_SUCCESS) { request->linkResult = new uint32_t[linkedBinary.size()]; - memcpy((void*)request->linkResult, linkedBinary.data(), + memcpy( + (void*)request->linkResult, + linkedBinary.data(), linkedBinary.size() * sizeof(uint32_t)); request->linkResultSize = linkedBinary.size(); } return success; } - catch(...) + catch (...) { return false; } diff --git a/tools/gfx/renderer-shared.cpp b/tools/gfx/renderer-shared.cpp index 5349049016..ae4ea3eefa 100644 --- a/tools/gfx/renderer-shared.cpp +++ b/tools/gfx/renderer-shared.cpp @@ -1178,8 +1178,8 @@ Result ShaderProgramBase::compileShaders(RendererBase* device) { ComPtr diagnosticsBlob; auto result = precompileService->getPrecompiledTargetCode( - compileTarget, - downstreamIR.writeRef(), + compileTarget, + downstreamIR.writeRef(), diagnosticsBlob.writeRef()); if (result == SLANG_OK) { diff --git a/tools/gfx/vulkan/glslang-module.cpp b/tools/gfx/vulkan/glslang-module.cpp index c2817cfce9..03afef7947 100644 --- a/tools/gfx/vulkan/glslang-module.cpp +++ b/tools/gfx/vulkan/glslang-module.cpp @@ -17,7 +17,8 @@ namespace gfx { using namespace Slang; -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! GlslangModule !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! GlslangModule +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Slang::Result GlslangModule::init() { diff --git a/tools/gfx/vulkan/glslang-module.h b/tools/gfx/vulkan/glslang-module.h index fedf7f33c9..cda9035779 100644 --- a/tools/gfx/vulkan/glslang-module.h +++ b/tools/gfx/vulkan/glslang-module.h @@ -1,12 +1,12 @@ // glslang-module.h #pragma once -#include "slang-com-ptr.h" -#include "slang-com-helper.h" #include "core/slang-list.h" -#include "slang.h" -#include "slang-glslang/slang-glslang.h" #include "external/spirv-tools/include/spirv-tools/linker.hpp" +#include "slang-com-helper.h" +#include "slang-com-ptr.h" +#include "slang-glslang/slang-glslang.h" +#include "slang.h" namespace gfx { diff --git a/tools/gfx/vulkan/vk-device.h b/tools/gfx/vulkan/vk-device.h index dc37137122..27cc4c3ce8 100644 --- a/tools/gfx/vulkan/vk-device.h +++ b/tools/gfx/vulkan/vk-device.h @@ -1,9 +1,9 @@ // vk-device.h #pragma once +#include "glslang-module.h" #include "vk-base.h" #include "vk-framebuffer.h" -#include "glslang-module.h" namespace gfx { diff --git a/tools/gfx/vulkan/vk-shader-program.cpp b/tools/gfx/vulkan/vk-shader-program.cpp index 7fe18c3af3..1627c95a7a 100644 --- a/tools/gfx/vulkan/vk-shader-program.cpp +++ b/tools/gfx/vulkan/vk-shader-program.cpp @@ -1,11 +1,10 @@ // vk-shader-program.cpp #include "vk-shader-program.h" +#include "external/spirv-tools/include/spirv-tools/linker.hpp" #include "vk-device.h" #include "vk-util.h" -#include "external/spirv-tools/include/spirv-tools/linker.hpp" - namespace gfx {