diff --git a/source/slang/glsl.meta.slang b/source/slang/glsl.meta.slang index 6f0ca1bf34..38e2871bd5 100644 --- a/source/slang/glsl.meta.slang +++ b/source/slang/glsl.meta.slang @@ -6409,6 +6409,7 @@ public property uint gl_SubgroupID public property uint gl_SubgroupSize { + [ForceInline] [require(cpp_cuda_glsl_hlsl_spirv_wgsl, subgroup_basic)] get { setupExtForSubgroupBasicBuiltIn(); @@ -6418,6 +6419,7 @@ public property uint gl_SubgroupSize public property uint gl_SubgroupInvocationID { + [ForceInline] [require(cpp_cuda_glsl_hlsl_spirv_wgsl, subgroup_basic)] get { setupExtForSubgroupBasicBuiltIn(); diff --git a/source/slang/hlsl.meta.slang b/source/slang/hlsl.meta.slang index 81b28b30a9..abc796cbe4 100644 --- a/source/slang/hlsl.meta.slang +++ b/source/slang/hlsl.meta.slang @@ -6,6 +6,12 @@ typedef uint UINT; __intrinsic_op($(kIROp_RequireGLSLExtension)) void __requireGLSLExtension(String extensionName); +__intrinsic_op($(kIROp_RequireWGSLExtension)) +void __requireWGSLExtension(String extensionName); + +__intrinsic_op($(kIROp_ImplicitSystemValue)) +uint __implicitSystemValue(String systemValueName); + //@public: /// Represents an interface for buffer data layout. /// This interface is used as a base for defining specific data layouts for buffers. @@ -15037,7 +15043,8 @@ uint WaveActiveCountBits(bool value) __glsl_extension(GL_KHR_shader_subgroup_basic) __spirv_version(1.3) [NonUniformReturn] -[require(cuda_glsl_hlsl_spirv, subgroup_basic)] +[ForceInline] +[require(cuda_glsl_hlsl_spirv_wgsl, subgroup_basic)] uint WaveGetLaneCount() { __target_switch @@ -15051,6 +15058,9 @@ uint WaveGetLaneCount() OpCapability GroupNonUniform; result:$$uint = OpLoad builtin(SubgroupSize:uint) }; + case wgsl: + __requireWGSLExtension("subgroups"); + return __implicitSystemValue("SV_WaveLaneCount"); } } @@ -15058,7 +15068,8 @@ uint WaveGetLaneCount() __glsl_extension(GL_KHR_shader_subgroup_basic) __spirv_version(1.3) [NonUniformReturn] -[require(cuda_glsl_hlsl_spirv, subgroup_basic)] +[ForceInline] +[require(cuda_glsl_hlsl_spirv_wgsl, subgroup_basic)] uint WaveGetLaneIndex() { __target_switch @@ -15072,6 +15083,9 @@ uint WaveGetLaneIndex() OpCapability GroupNonUniform; result:$$uint = OpLoad builtin(SubgroupLocalInvocationId:uint) }; + case wgsl: + __requireWGSLExtension("subgroups"); + return __implicitSystemValue("SV_WaveLaneIndex"); } } diff --git a/source/slang/slang-emit-c-like.cpp b/source/slang/slang-emit-c-like.cpp index db2c0150f7..102159cf68 100644 --- a/source/slang/slang-emit-c-like.cpp +++ b/source/slang/slang-emit-c-like.cpp @@ -3055,6 +3055,11 @@ void CLikeSourceEmitter::defaultEmitInstExpr(IRInst* inst, const EmitOpInfo& inO emitOperand(as<IRGlobalValueRef>(inst)->getOperand(0), getInfo(EmitOp::General)); break; } + case kIROp_RequireWGSLExtension: + { + emitRequireExtension(inst); + break; + } default: diagnoseUnhandledInst(inst); break; diff --git a/source/slang/slang-emit-c-like.h b/source/slang/slang-emit-c-like.h index e83b6e5861..6040f3b302 100644 --- a/source/slang/slang-emit-c-like.h +++ b/source/slang/slang-emit-c-like.h @@ -678,6 +678,8 @@ class CLikeSourceEmitter : public SourceEmitterBase void _emitCallArgList(IRCall* call, int startingOperandIndex = 1); virtual void emitCallArg(IRInst* arg); + virtual void emitRequireExtension(IRInst* inst) { SLANG_UNUSED(inst); } + String _generateUniqueName(const UnownedStringSlice& slice); // Sort witnessTable entries according to the order defined in the witnessed interface type. diff --git a/source/slang/slang-emit-glsl.cpp b/source/slang/slang-emit-glsl.cpp index 25dab3fb35..11f7b5ad5a 100644 --- a/source/slang/slang-emit-glsl.cpp +++ b/source/slang/slang-emit-glsl.cpp @@ -47,7 +47,7 @@ void GLSLSourceEmitter::_beforeComputeEmitProcessInstruction( // // Handle cases where "require" IR operations exist in the function body and are required // as entry point decorations. - auto entryPoints = getReferencingEntryPoints(m_referencingEntryPoints, parentFunc); + auto entryPoints = m_callGraph.getReferencingEntryPoints(parentFunc); if (entryPoints == nullptr) return; @@ -81,7 +81,7 @@ void GLSLSourceEmitter::_beforeComputeEmitProcessInstruction( void GLSLSourceEmitter::beforeComputeEmitActions(IRModule* module) { - buildEntryPointReferenceGraph(this->m_referencingEntryPoints, module); + m_callGraph.build(module); IRBuilder builder(module); for (auto globalInst : module->getGlobalInsts()) diff --git a/source/slang/slang-emit-glsl.h b/source/slang/slang-emit-glsl.h index 8308a9954d..7ea37f81f0 100644 --- a/source/slang/slang-emit-glsl.h +++ b/source/slang/slang-emit-glsl.h @@ -4,6 +4,7 @@ #include "slang-emit-c-like.h" #include "slang-extension-tracker.h" +#include "slang-ir-call-graph.h" namespace Slang { @@ -178,7 +179,7 @@ class GLSLSourceEmitter : public CLikeSourceEmitter void _beforeComputeEmitProcessInstruction(IRInst* parentFunc, IRInst* inst, IRBuilder& builder); - Dictionary<IRInst*, HashSet<IRFunc*>> m_referencingEntryPoints; + CallGraph m_callGraph; RefPtr<ShaderExtensionTracker> m_glslExtensionTracker; }; diff --git a/source/slang/slang-emit-spirv.cpp b/source/slang/slang-emit-spirv.cpp index ef015df7f9..5a65e2d61b 100644 --- a/source/slang/slang-emit-spirv.cpp +++ b/source/slang/slang-emit-spirv.cpp @@ -3553,8 +3553,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex { auto parentFunc = getParentFunc(inst); - HashSet<IRFunc*>* entryPointsUsingInst = - getReferencingEntryPoints(m_referencingEntryPoints, parentFunc); + auto entryPointsUsingInst = m_callGraph.getReferencingEntryPoints(parentFunc); for (IRFunc* entryPoint : *entryPointsUsingInst) { bool isQuad = true; @@ -3608,8 +3607,11 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex } case kIROp_RequireMaximallyReconverges: - if (auto entryPointsUsingInst = - getReferencingEntryPoints(m_referencingEntryPoints, getParentFunc(inst))) + if ( + // auto entryPointsUsingInst = + // getReferencingEntryPoints(m_referencingEntryPoints, getParentFunc(inst)) + auto entryPointsUsingInst = + m_callGraph.getReferencingEntryPoints(getParentFunc(inst))) { ensureExtensionDeclaration(UnownedStringSlice("SPV_KHR_maximal_reconvergence")); for (IRFunc* entryPoint : *entryPointsUsingInst) @@ -3623,7 +3625,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex break; case kIROp_RequireQuadDerivatives: if (auto entryPointsUsingInst = - getReferencingEntryPoints(m_referencingEntryPoints, getParentFunc(inst))) + m_callGraph.getReferencingEntryPoints(getParentFunc(inst))) { ensureExtensionDeclaration(UnownedStringSlice("SPV_KHR_quad_control")); requireSPIRVCapability(SpvCapabilityQuadControlKHR); @@ -4326,7 +4328,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex if (m_mapIRInstToSpvInst.tryGetValue(globalInst, spvGlobalInst)) { // Is this globalInst referenced by this entry point? - auto refSet = m_referencingEntryPoints.tryGetValue(globalInst); + auto refSet = m_callGraph.getReferencingEntryPoints(globalInst); if (refSet && refSet->contains(entryPoint)) { if (!isSpirv14OrLater()) @@ -5129,7 +5131,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex bool isInstUsedInStage(IRInst* inst, Stage s) { - auto* referencingEntryPoints = m_referencingEntryPoints.tryGetValue(inst); + auto referencingEntryPoints = m_callGraph.getReferencingEntryPoints(inst); if (!referencingEntryPoints) return false; for (auto entryPoint : *referencingEntryPoints) @@ -5329,7 +5331,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex } else if (semanticName == "sv_primitiveid") { - auto entryPoints = m_referencingEntryPoints.tryGetValue(inst); + auto entryPoints = m_callGraph.getReferencingEntryPoints(inst); // SPIRV requires `Geometry` capability being declared for a fragment // shader, if that shader uses sv_primitiveid. // We will check if this builtin is used by non-ray-tracing, non-geometry or @@ -8029,7 +8031,7 @@ struct SPIRVEmitContext : public SourceEmitterBase, public SPIRVEmitSharedContex case SpvOpExecutionMode: { if (auto refEntryPointSet = - m_referencingEntryPoints.tryGetValue(getParentFunc(inst))) + m_callGraph.getReferencingEntryPoints(getParentFunc(inst))) { for (auto entryPoint : *refEntryPointSet) { diff --git a/source/slang/slang-emit-wgsl.cpp b/source/slang/slang-emit-wgsl.cpp index 13c79e9acc..933e750e2a 100644 --- a/source/slang/slang-emit-wgsl.cpp +++ b/source/slang/slang-emit-wgsl.cpp @@ -1696,4 +1696,9 @@ void WGSLSourceEmitter::handleRequiredCapabilitiesImpl(IRInst* inst) } } +void WGSLSourceEmitter::emitRequireExtension(IRInst* inst) +{ + _requireExtension(as<IRRequireWGSLExtension>(inst)->getExtensionName()); +} + } // namespace Slang diff --git a/source/slang/slang-emit-wgsl.h b/source/slang/slang-emit-wgsl.h index 441933b570..2392c4d3c3 100644 --- a/source/slang/slang-emit-wgsl.h +++ b/source/slang/slang-emit-wgsl.h @@ -65,6 +65,8 @@ class WGSLSourceEmitter : public CLikeSourceEmitter virtual RefObject* getExtensionTracker() SLANG_OVERRIDE { return m_extensionTracker; } + virtual void emitRequireExtension(IRInst* inst) SLANG_OVERRIDE; + private: bool maybeEmitSystemSemantic(IRInst* inst); diff --git a/source/slang/slang-ir-call-graph.cpp b/source/slang/slang-ir-call-graph.cpp index 47b18be2ed..9677e55386 100644 --- a/source/slang/slang-ir-call-graph.cpp +++ b/source/slang/slang-ir-call-graph.cpp @@ -6,14 +6,45 @@ namespace Slang { -void buildEntryPointReferenceGraph( - Dictionary<IRInst*, HashSet<IRFunc*>>& referencingEntryPoints, - IRModule* module) +CallGraph::CallGraph(IRModule* module) +{ + build(module); +} + +template<typename T, typename U> +static void addToReferenceMap(Dictionary<T, HashSet<U>>& map, T key, U value) +{ + if (auto set = map.tryGetValue(key)) + { + set->add(value); + } + else + { + HashSet<U> newSet; + newSet.add(value); + map.add(key, _Move(newSet)); + } +} + + +void CallGraph::registerInstructionReference(IRInst* inst, IRFunc* entryPoint, IRFunc* parentFunc) +{ + addToReferenceMap(m_referencingEntryPoints, inst, entryPoint); + addToReferenceMap(m_referencingFunctions, inst, parentFunc); +} + +void CallGraph::registerCallReference(IRFunc* func, IRCall* call) +{ + addToReferenceMap(m_referencingCalls, func, call); +} + +void CallGraph::build(IRModule* module) { struct WorkItem { - IRFunc* entryPoint; IRInst* inst; + IRFunc* entryPoint; + IRFunc* parentFunc; HashCode getHashCode() const { @@ -32,51 +63,52 @@ void buildEntryPointReferenceGraph( workList.add(item); }; - auto registerEntryPointReference = [&](IRFunc* entryPoint, IRInst* inst) - { - if (auto set = referencingEntryPoints.tryGetValue(inst)) - set->add(entryPoint); - else - { - HashSet<IRFunc*> newSet; - newSet.add(entryPoint); - referencingEntryPoints.add(inst, _Move(newSet)); - } - }; - auto visit = [&](IRFunc* entryPoint, IRInst* inst) + auto visit = [&](IRInst* inst, IRFunc* entryPoint, IRFunc* parentFunc) { if (auto code = as<IRGlobalValueWithCode>(inst)) { - registerEntryPointReference(entryPoint, inst); + registerInstructionReference(inst, entryPoint, parentFunc); + + if (auto func = as<IRFunc>(code)) + { + parentFunc = func; + } + for (auto child : code->getChildren()) { - addToWorkList({entryPoint, child}); + addToWorkList({child, entryPoint, parentFunc}); } return; } + switch (inst->getOp()) { + // Only these instruction types and `IRGlobalValueWithCode` instructions are registered to + // the reference graph. + case kIROp_Call: + { + auto call = as<IRCall>(inst); + registerCallReference(as<IRFunc>(call->getCallee()), call); + addToWorkList({call->getCallee(), entryPoint, parentFunc}); + } + [[fallthrough]]; case kIROp_GlobalParam: case kIROp_SPIRVAsmOperandBuiltinVar: - registerEntryPointReference(entryPoint, inst); + case kIROp_ImplicitSystemValue: + registerInstructionReference(inst, entryPoint, parentFunc); break; + case kIROp_Block: case kIROp_SPIRVAsm: for (auto child : inst->getChildren()) { - addToWorkList({entryPoint, child}); - } - break; - case kIROp_Call: - { - auto call = as<IRCall>(inst); - addToWorkList({entryPoint, call->getCallee()}); + addToWorkList({child, entryPoint, parentFunc}); } break; case kIROp_SPIRVAsmOperandInst: { auto operand = as<IRSPIRVAsmOperandInst>(inst); - addToWorkList({entryPoint, operand->getValue()}); + addToWorkList({operand->getValue(), entryPoint, parentFunc}); } break; } @@ -88,7 +120,7 @@ void buildEntryPointReferenceGraph( case kIROp_GlobalParam: case kIROp_GlobalVar: case kIROp_SPIRVAsmOperandBuiltinVar: - addToWorkList({entryPoint, operand}); + addToWorkList({operand, entryPoint, parentFunc}); break; } } @@ -99,21 +131,41 @@ void buildEntryPointReferenceGraph( if (globalInst->getOp() == kIROp_Func && globalInst->findDecoration<IREntryPointDecoration>()) { - visit(as<IRFunc>(globalInst), globalInst); + auto entryPointFunc = as<IRFunc>(globalInst); + visit(globalInst, entryPointFunc, nullptr); } } for (Index i = 0; i < workList.getCount(); i++) - visit(workList[i].entryPoint, workList[i].inst); + visit(workList[i].inst, workList[i].entryPoint, workList[i].parentFunc); } -HashSet<IRFunc*>* getReferencingEntryPoints( - Dictionary<IRInst*, HashSet<IRFunc*>>& m_referencingEntryPoints, - IRInst* inst) +const HashSet<IRFunc*>* CallGraph::getReferencingEntryPoints(IRInst* inst) const { - auto* referencingEntryPoints = m_referencingEntryPoints.tryGetValue(inst); + const auto* referencingEntryPoints = m_referencingEntryPoints.tryGetValue(inst); if (!referencingEntryPoints) return nullptr; return referencingEntryPoints; } +const HashSet<IRFunc*>* CallGraph::getReferencingFunctions(IRInst* inst) const +{ + const auto* referencingFunctions = m_referencingFunctions.tryGetValue(inst); + if (!referencingFunctions) + return nullptr; + return referencingFunctions; +} + +const HashSet<IRCall*>* CallGraph::getReferencingCalls(IRFunc* func) const +{ + const auto* referencingCalls = m_referencingCalls.tryGetValue(func); + if (!referencingCalls) + return nullptr; + return referencingCalls; +} + +const Dictionary<IRInst*, HashSet<IRFunc*>>& CallGraph::getReferencingEntryPointsMap() const +{ + return m_referencingEntryPoints; +} + } // namespace Slang diff --git a/source/slang/slang-ir-call-graph.h b/source/slang/slang-ir-call-graph.h index 4ee6423566..e6de6af121 100644 --- a/source/slang/slang-ir-call-graph.h +++ b/source/slang/slang-ir-call-graph.h @@ -1,15 +1,40 @@ +#pragma once + #include "slang-ir-clone.h" #include "slang-ir-insts.h" namespace Slang { -void buildEntryPointReferenceGraph( - Dictionary<IRInst*, HashSet<IRFunc*>>& referencingEntryPoints, - IRModule* module); +struct CallGraph +{ +public: + CallGraph() = default; + explicit CallGraph(IRModule* module); + + void build(IRModule* module); + + /// Retrieves the set of entry points that transitively invoke the given instruction. + /// Returns nullptr if the instruction has no referencing entry points. + const HashSet<IRFunc*>* getReferencingEntryPoints(IRInst* inst) const; + + /// Retrieves the set of functions that directly contain the given instruction in their body. + /// Returns nullptr if the instruction is not referenced by any function. + const HashSet<IRFunc*>* getReferencingFunctions(IRInst* inst) const; + + /// Retrieves the set of calls that invoke the given function. + /// Returns nullptr if the function is never called. + const HashSet<IRCall*>* getReferencingCalls(IRFunc* func) const; + + const Dictionary<IRInst*, HashSet<IRFunc*>>& getReferencingEntryPointsMap() const; + +private: + void registerInstructionReference(IRInst* inst, IRFunc* entryPoint, IRFunc* parentFunc); + void registerCallReference(IRFunc* func, IRCall* call); -HashSet<IRFunc*>* getReferencingEntryPoints( - Dictionary<IRInst*, HashSet<IRFunc*>>& m_referencingEntryPoints, - IRInst* inst); + Dictionary<IRInst*, HashSet<IRFunc*>> m_referencingEntryPoints; + Dictionary<IRInst*, HashSet<IRFunc*>> m_referencingFunctions; + Dictionary<IRFunc*, HashSet<IRCall*>> m_referencingCalls; +}; } // namespace Slang diff --git a/source/slang/slang-ir-inst-defs.h b/source/slang/slang-ir-inst-defs.h index 55880eab5d..20c4583bde 100644 --- a/source/slang/slang-ir-inst-defs.h +++ b/source/slang/slang-ir-inst-defs.h @@ -667,10 +667,16 @@ INST(discard, discard, 0, 0) INST(RequirePrelude, RequirePrelude, 1, 0) INST(RequireGLSLExtension, RequireGLSLExtension, 1, 0) +INST(RequireWGSLExtension, RequireWGSLExtension, 1, 0) INST(RequireComputeDerivative, RequireComputeDerivative, 0, 0) INST(StaticAssert, StaticAssert, 2, 0) INST(Printf, Printf, 1, 0) +// Built-in inputs/outputs(system values) that are implicitly added. +// These must be passed in as entry function parameters for the target language(eg. WGSL and Metal), but +// do not explicitly originate from decorated entry point function parameters in Slang. +INST(ImplicitSystemValue, ImplicitSystemValue, 1, 0) + // Quad control execution modes. INST(RequireMaximallyReconverges, RequireMaximallyReconverges, 0, 0) INST(RequireQuadDerivatives, RequireQuadDerivatives, 0, 0) diff --git a/source/slang/slang-ir-insts.h b/source/slang/slang-ir-insts.h index dbefa68c7e..8366ff76a8 100644 --- a/source/slang/slang-ir-insts.h +++ b/source/slang/slang-ir-insts.h @@ -3479,6 +3479,16 @@ struct IRRequireGLSLExtension : IRInst } }; +struct IRRequireWGSLExtension : IRInst +{ + IR_LEAF_ISA(RequireWGSLExtension) + UnownedStringSlice getExtensionName() + { + return as<IRStringLit>(getOperand(0))->getStringSlice(); + } +}; +; + struct IRRequireComputeDerivative : IRInst { IR_LEAF_ISA(RequireComputeDerivative) @@ -3499,6 +3509,15 @@ struct IRStaticAssert : IRInst IR_LEAF_ISA(StaticAssert) }; +struct IRImplicitSystemValue : IRInst +{ + IR_LEAF_ISA(ImplicitSystemValue) + UnownedStringSlice getSystemValueName() + { + return as<IRStringLit>(getOperand(0))->getStringSlice(); + } +}; + struct IREmbeddedDownstreamIR : IRInst { IR_LEAF_ISA(EmbeddedDownstreamIR) diff --git a/source/slang/slang-ir-legalize-system-values.cpp b/source/slang/slang-ir-legalize-system-values.cpp new file mode 100644 index 0000000000..92b4aa1530 --- /dev/null +++ b/source/slang/slang-ir-legalize-system-values.cpp @@ -0,0 +1,239 @@ +#include "slang-ir-legalize-system-values.h" + +#include "core/slang-dictionary.h" +#include "core/slang-string.h" +#include "slang-ir-call-graph.h" +#include "slang-ir-insts.h" +#include "slang-ir-legalize-varying-params.h" + +namespace Slang +{ + +class ImplicitSystemValueLegalizationContext +{ +public: + ImplicitSystemValueLegalizationContext( + IRModule* module, + const CallGraph& callGraph, + const List<IRImplicitSystemValue*>& implicitSystemValueInstructions) + : m_callGraph(callGraph) + , m_implicitSystemValueInstructions(implicitSystemValueInstructions) + , m_builder(module) + , m_paramType(m_builder.getUIntType()) + { + } + + void legalize() + { + for (auto implicitSysVal : m_implicitSystemValueInstructions) + { + // Call graph is guaranteed to return valid referencing functions(non nullptr) as + // instructions processed here are all valid/non-dead instructions. + for (auto parentFunc : *m_callGraph.getReferencingFunctions(implicitSysVal)) + { + auto param = getOrCreateSystemValueVariable(parentFunc, implicitSysVal); + implicitSysVal->replaceUsesWith(param); + implicitSysVal->removeAndDeallocate(); + } + } + } + +private: + // + // A function (including entry points) must have at most one parameter for each implicit system + // value semantic type. + // + // This map tracks the association between system value semantics and their corresponding + // function parameters. + // + using SystemValueParamMap = Dictionary<SystemValueSemanticName, IRParam*>; + SystemValueParamMap& getParamMap(IRFunc* func) + { + if (auto map = m_functionMap.tryGetValue(func)) + { + return *map; + } + else + { + m_functionMap.add(func, SystemValueParamMap()); + return m_functionMap.getValue(func); + } + } + + // + // Attempt to retrieve a parameter for a specific function and system value type combination. + // Returns nullptr if parameter has not been created. + // + IRParam* tryGetParam(IRFunc* func, SystemValueSemanticName systemValueName) + { + if (auto param = getParamMap(func).tryGetValue(systemValueName)) + { + return *param; + } + else + { + return nullptr; + } + } + + struct ModifyCallWorkItem + { + IRCall* call; + IRFunc* caller; + }; + + // + // Implicit system values are "global variables" and can be used anywhere within the source + // code. The implementation target(i.e WGSL) however requires system values, aka built-in + // values, to be accessed via parameters to the entry point; they are not globally available. + // + // For any implicit system values found in non entry point functions, we need to ensure that + // they are explicitly passed as parameters from the entry point to the relevant functions. This + // means adding new parameters to the function signatures to include the required system values. + // + // This function traverses the call graph of a function that contains an implicit system value + // instruction, and adds necessary parameters to pass in the system value variable up to the + // entry point function. Returns work items of calls that need to be modified as a result of + // adding the parameters. + // + List<ModifyCallWorkItem> createFunctionParams( + IRFunc* func, + SystemValueSemanticName systemValueName, + UnownedStringSlice systemValueString) + { + List<IRFunc*> createParamWorkList; + List<ModifyCallWorkItem> modifyCallWorkList; + + const auto addWorkItems = [&](const HashSet<IRCall*>& calls) + { + for (auto call : calls) + { + for (auto caller : *m_callGraph.getReferencingFunctions(call)) + { + // The caller(of a function that was added a parameter) also requires a + // new parameter to pass in the system value variable to the callee. + createParamWorkList.add(caller); + + // The call needs to be modified to account for the new parameter. + modifyCallWorkList.add({call, caller}); + } + } + }; + + const auto createParamWork = [&](IRFunc* func) + { + // If the parameter for system value type has not been created, create it. + if (!tryGetParam(func, systemValueName)) + { + m_builder.setInsertBefore(func->getFirstBlock()->getFirstOrdinaryInst()); + + auto param = m_builder.emitParam(m_paramType); + + // Add system value semantic decoration if adding to entry point. + if (func->findDecoration<IREntryPointDecoration>()) + { + m_builder.addSemanticDecoration(param, systemValueString); + } + + fixUpFuncType(func); + getParamMap(func).add(systemValueName, param); + + // Update all functions that call this function. + if (auto calls = m_callGraph.getReferencingCalls(func)) + { + addWorkItems(*calls); + } + } + }; + + createParamWorkList.add(func); + for (Index i = 0; i < createParamWorkList.getCount(); i++) + { + createParamWork(createParamWorkList[i]); + } + + return modifyCallWorkList; + } + + // + // The function calls need to be modified to account for the change in function signature. + // + void modifyCalls( + const List<ModifyCallWorkItem>& workList, + SystemValueSemanticName systemValueName) + { + for (const auto workItem : workList) + { + auto call = workItem.call; + auto param = tryGetParam(workItem.caller, systemValueName); + SLANG_ASSERT(param); + + List<IRInst*> newCallParams; + for (auto arg : call->getArgsList()) + { + newCallParams.add(arg); + } + newCallParams.add(param); + + m_builder.setInsertAfter(call); + auto newCall = m_builder.emitCallInst(m_paramType, call->getCallee(), newCallParams); + + call->replaceUsesWith(newCall); + call->transferDecorationsTo(newCall); + call->removeAndDeallocate(); + } + } + + IRParam* getOrCreateSystemValueVariable( + IRFunc* parentFunc, + IRImplicitSystemValue* implicitSysVal) + { + auto systemValueName = + convertSystemValueSemanticNameToEnum(implicitSysVal->getSystemValueName()); + + // Implicit system values are currently only being used for subgroup size and + // subgroup invocation id. + SLANG_ASSERT( + (systemValueName == SystemValueSemanticName::WaveLaneCount) || + (systemValueName == SystemValueSemanticName::WaveLaneIndex)); + + // If parameter for the specific function and system value type combination was already + // created, return it directly. + if (auto existingParam = tryGetParam(parentFunc, systemValueName)) + return existingParam; + + // Create new parameters for the relevant functions up to the entry point function. + const auto callWorkItems = + createFunctionParams(parentFunc, systemValueName, implicitSysVal->getSystemValueName()); + + // Modify related function calls to account for the new parameters. + modifyCalls(callWorkItems, systemValueName); + + auto newParam = tryGetParam(parentFunc, systemValueName); + SLANG_ASSERT(newParam); + return newParam; + } + + const CallGraph& m_callGraph; + const List<IRImplicitSystemValue*>& m_implicitSystemValueInstructions; + + Dictionary<IRFunc*, SystemValueParamMap> m_functionMap; + IRBuilder m_builder; + + // Type of system value. + // + // Implicit system values are currently only being used for subgroup size and + // subgroup invocation id, both of which are 32-bit unsigned. + IRType* m_paramType; +}; + +void legalizeImplicitSystemValues( + IRModule* module, + const CallGraph& callGraph, + const List<IRImplicitSystemValue*>& implicitSystemValueInstructions) +{ + ImplicitSystemValueLegalizationContext(module, callGraph, implicitSystemValueInstructions) + .legalize(); +} + +} // namespace Slang diff --git a/source/slang/slang-ir-legalize-system-values.h b/source/slang/slang-ir-legalize-system-values.h new file mode 100644 index 0000000000..4a142da140 --- /dev/null +++ b/source/slang/slang-ir-legalize-system-values.h @@ -0,0 +1,14 @@ +// slang-ir-legalize-system-values.h +#pragma once +#include "slang-ir-call-graph.h" +#include "slang-ir-insts.h" + +namespace Slang +{ + +void legalizeImplicitSystemValues( + IRModule* module, + const CallGraph& callGraph, + const List<IRImplicitSystemValue*>& implicitSystemValueInstructions); + +} // namespace Slang diff --git a/source/slang/slang-ir-legalize-varying-params.cpp b/source/slang/slang-ir-legalize-varying-params.cpp index 3b65ee59af..05d640cee4 100644 --- a/source/slang/slang-ir-legalize-varying-params.cpp +++ b/source/slang/slang-ir-legalize-varying-params.cpp @@ -3854,6 +3854,20 @@ class LegalizeWGSLEntryPointContext : public LegalizeShaderEntryPointContext break; } + case SystemValueSemanticName::WaveLaneCount: + { + result.systemValueName = toSlice("subgroup_size"); + result.permittedTypes.add(builder.getUIntType()); + break; + } + + case SystemValueSemanticName::WaveLaneIndex: + { + result.systemValueName = toSlice("subgroup_invocation_id"); + result.permittedTypes.add(builder.getUIntType()); + break; + } + default: { m_sink->diagnose( diff --git a/source/slang/slang-ir-legalize-varying-params.h b/source/slang/slang-ir-legalize-varying-params.h index e742f30936..0a7c3be8e7 100644 --- a/source/slang/slang-ir-legalize-varying-params.h +++ b/source/slang/slang-ir-legalize-varying-params.h @@ -68,6 +68,8 @@ void depointerizeInputParams(IRFunc* entryPoint); M(Target, SV_Target) \ M(StartVertexLocation, SV_StartVertexLocation) \ M(StartInstanceLocation, SV_StartInstanceLocation) \ + M(WaveLaneCount, SV_WaveLaneCount) \ + M(WaveLaneIndex, SV_WaveLaneIndex) \ /* end */ /// A known system-value semantic name that can be applied to a parameter diff --git a/source/slang/slang-ir-specialize-stage-switch.cpp b/source/slang/slang-ir-specialize-stage-switch.cpp index f65aa4d4cd..f984be9c1e 100644 --- a/source/slang/slang-ir-specialize-stage-switch.cpp +++ b/source/slang/slang-ir-specialize-stage-switch.cpp @@ -123,8 +123,7 @@ void specializeFuncToStage( void specializeStageSwitch(IRModule* module) { - Dictionary<IRInst*, HashSet<IRFunc*>> mapInstToReferencingEntryPoints; - buildEntryPointReferenceGraph(mapInstToReferencingEntryPoints, module); + const auto callGraph = CallGraph(module); HashSet<IRInst*> stageSpecificFunctions; discoverStageSpecificFunctions(stageSpecificFunctions, module); @@ -133,7 +132,7 @@ void specializeStageSwitch(IRModule* module) Dictionary<IRInst*, Dictionary<Stage, IRInst*>> mapFuncToStageSpecializedFunc; for (auto func : stageSpecificFunctions) { - auto referencingEntryPoints = mapInstToReferencingEntryPoints.tryGetValue(func); + auto referencingEntryPoints = callGraph.getReferencingEntryPoints(func); if (!referencingEntryPoints) continue; if (func->findDecoration<IREntryPointDecoration>()) diff --git a/source/slang/slang-ir-spirv-legalize.cpp b/source/slang/slang-ir-spirv-legalize.cpp index c672180b70..e670f4a730 100644 --- a/source/slang/slang-ir-spirv-legalize.cpp +++ b/source/slang/slang-ir-spirv-legalize.cpp @@ -2,7 +2,6 @@ #include "slang-ir-spirv-legalize.h" #include "slang-emit-base.h" -#include "slang-ir-call-graph.h" #include "slang-ir-clone.h" #include "slang-ir-composite-reg-to-mem.h" #include "slang-ir-dce.h" @@ -2102,7 +2101,7 @@ static bool hasExplicitInterlockInst(IRFunc* func) void insertFragmentShaderInterlock(SPIRVEmitSharedContext* context, IRModule* module) { HashSet<IRFunc*> fragmentShaders; - for (auto& [inst, entryPoints] : context->m_referencingEntryPoints) + for (const auto& [inst, entryPoints] : context->m_callGraph.getReferencingEntryPointsMap()) { if (isRasterOrderedResource(inst)) { @@ -2154,7 +2153,7 @@ void legalizeIRForSPIRV( SLANG_UNUSED(entryPoints); legalizeSPIRV(context, module, codeGenContext->getSink()); simplifyIRForSpirvLegalization(context->m_targetProgram, codeGenContext->getSink(), module); - buildEntryPointReferenceGraph(context->m_referencingEntryPoints, module); + context->m_callGraph.build(module); insertFragmentShaderInterlock(context, module); } diff --git a/source/slang/slang-ir-spirv-legalize.h b/source/slang/slang-ir-spirv-legalize.h index 3c9bdf26a5..01adc7b604 100644 --- a/source/slang/slang-ir-spirv-legalize.h +++ b/source/slang/slang-ir-spirv-legalize.h @@ -1,6 +1,7 @@ // slang-ir-spirv-legalize.h #pragma once #include "../core/slang-basic.h" +#include "slang-ir-call-graph.h" #include "slang-ir-insts.h" #include "slang-ir-spirv-snippet.h" @@ -20,9 +21,8 @@ struct SPIRVEmitSharedContext TargetProgram* m_targetProgram; Dictionary<IRTargetIntrinsicDecoration*, RefPtr<SpvSnippet>> m_parsedSpvSnippets; - Dictionary<IRInst*, HashSet<IRFunc*>> - m_referencingEntryPoints; // The entry-points that directly or transitively reference this - // global inst. + // Track entry-points that directly or transitively reference this global inst. + CallGraph m_callGraph; DiagnosticSink* m_sink; const SPIRVCoreGrammarInfo* m_grammarInfo; diff --git a/source/slang/slang-ir-translate-glsl-global-var.cpp b/source/slang/slang-ir-translate-glsl-global-var.cpp index 7b2b8d1ee2..5912ccef96 100644 --- a/source/slang/slang-ir-translate-glsl-global-var.cpp +++ b/source/slang/slang-ir-translate-glsl-global-var.cpp @@ -13,8 +13,7 @@ struct GlobalVarTranslationContext void processModule(IRModule* module) { - Dictionary<IRInst*, HashSet<IRFunc*>> referencingEntryPoints; - buildEntryPointReferenceGraph(referencingEntryPoints, module); + const auto callGraph = CallGraph(module); List<IRInst*> entryPoints; List<IRInst*> getWorkGroupSizeInsts; @@ -30,7 +29,7 @@ struct GlobalVarTranslationContext getWorkGroupSizeInsts.add(inst); } for (auto inst : getWorkGroupSizeInsts) - materializeGetWorkGroupSize(module, referencingEntryPoints, inst); + materializeGetWorkGroupSize(module, callGraph, inst); IRBuilder builder(module); for (auto entryPoint : entryPoints) @@ -39,7 +38,7 @@ struct GlobalVarTranslationContext List<IRInst*> inputVars; for (auto inst : module->getGlobalInsts()) { - if (auto referencingEntryPointSet = referencingEntryPoints.tryGetValue(inst)) + if (auto referencingEntryPointSet = callGraph.getReferencingEntryPoints(inst)) { if (referencingEntryPointSet->contains((IRFunc*)entryPoint)) { @@ -266,7 +265,7 @@ struct GlobalVarTranslationContext // void materializeGetWorkGroupSize( IRModule* module, - Dictionary<IRInst*, HashSet<IRFunc*>>& referenceGraph, + const CallGraph& callGraph, IRInst* workgroupSizeInst) { IRBuilder builder(workgroupSizeInst); @@ -276,7 +275,7 @@ struct GlobalVarTranslationContext { if (auto parentFunc = getParentFunc(use->getUser())) { - auto referenceSet = referenceGraph.tryGetValue(parentFunc); + auto referenceSet = callGraph.getReferencingEntryPoints(parentFunc); if (!referenceSet) return; if (referenceSet->getCount() == 1) diff --git a/source/slang/slang-ir-wgsl-legalize.cpp b/source/slang/slang-ir-wgsl-legalize.cpp index efa028703c..75b49f013e 100644 --- a/source/slang/slang-ir-wgsl-legalize.cpp +++ b/source/slang/slang-ir-wgsl-legalize.cpp @@ -3,6 +3,7 @@ #include "slang-ir-insts.h" #include "slang-ir-legalize-binary-operator.h" #include "slang-ir-legalize-global-values.h" +#include "slang-ir-legalize-system-values.h" #include "slang-ir-legalize-varying-params.h" #include "slang-ir.h" @@ -121,52 +122,63 @@ static void legalizeSwitch(IRSwitch* switchInst) switchInst->removeAndDeallocate(); } -static void processInst(IRInst* inst) +class InstructionLegalizationContext { - switch (inst->getOp()) +public: + void processInst(IRInst* inst) { - case kIROp_Call: - legalizeCall(static_cast<IRCall*>(inst)); - break; - - case kIROp_Switch: - legalizeSwitch(as<IRSwitch>(inst)); - break; - - // For all binary operators, make sure both side of the operator have the same type - // (vector-ness and matrix-ness). - case kIROp_Add: - case kIROp_Sub: - case kIROp_Mul: - case kIROp_Div: - case kIROp_FRem: - case kIROp_IRem: - case kIROp_And: - case kIROp_Or: - case kIROp_BitAnd: - case kIROp_BitOr: - case kIROp_BitXor: - case kIROp_Lsh: - case kIROp_Rsh: - case kIROp_Eql: - case kIROp_Neq: - case kIROp_Greater: - case kIROp_Less: - case kIROp_Geq: - case kIROp_Leq: - legalizeBinaryOp(inst); - break; - - case kIROp_Func: - legalizeFunc(static_cast<IRFunc*>(inst)); - [[fallthrough]]; - default: - for (auto child : inst->getModifiableChildren()) + switch (inst->getOp()) { - processInst(child); + case kIROp_Call: + legalizeCall(static_cast<IRCall*>(inst)); + break; + + case kIROp_Switch: + legalizeSwitch(as<IRSwitch>(inst)); + break; + + // For all binary operators, make sure both side of the operator have the same type + // (vector-ness and matrix-ness). + case kIROp_Add: + case kIROp_Sub: + case kIROp_Mul: + case kIROp_Div: + case kIROp_FRem: + case kIROp_IRem: + case kIROp_And: + case kIROp_Or: + case kIROp_BitAnd: + case kIROp_BitOr: + case kIROp_BitXor: + case kIROp_Lsh: + case kIROp_Rsh: + case kIROp_Eql: + case kIROp_Neq: + case kIROp_Greater: + case kIROp_Less: + case kIROp_Geq: + case kIROp_Leq: + legalizeBinaryOp(inst); + break; + + case kIROp_ImplicitSystemValue: + implicitSystemValueInstructions.add(as<IRImplicitSystemValue>(inst)); + break; + + case kIROp_Func: + legalizeFunc(static_cast<IRFunc*>(inst)); + [[fallthrough]]; + + default: + for (auto child : inst->getModifiableChildren()) + { + processInst(child); + } } } -} + + List<IRImplicitSystemValue*> implicitSystemValueInstructions; +}; struct GlobalInstInliningContext : public GlobalInstInliningContextGeneric { @@ -215,10 +227,21 @@ void legalizeIRForWGSL(IRModule* module, DiagnosticSink* sink) entryPoints.add(info); } - legalizeEntryPointVaryingParamsForWGSL(module, sink, entryPoints); - // Go through every instruction in the module and legalize them as needed. - processInst(module->getModuleInst()); + InstructionLegalizationContext instContext; + instContext.processInst(module->getModuleInst()); + + // Legalize implicit system values to entry point parameters. + if (instContext.implicitSystemValueInstructions.getCount() != 0) + { + const auto callGraph = CallGraph(module); + legalizeImplicitSystemValues( + module, + callGraph, + instContext.implicitSystemValueInstructions); + } + + legalizeEntryPointVaryingParamsForWGSL(module, sink, entryPoints); // Some global insts are illegal, e.g. function calls. // We need to inline and remove those. diff --git a/tests/glsl-intrinsic/shader-subgroup/shader-subgroup-builtin-variables.slang b/tests/glsl-intrinsic/shader-subgroup/shader-subgroup-builtin-variables.slang index 21b533178e..626a613a4e 100644 --- a/tests/glsl-intrinsic/shader-subgroup/shader-subgroup-builtin-variables.slang +++ b/tests/glsl-intrinsic/shader-subgroup/shader-subgroup-builtin-variables.slang @@ -10,6 +10,7 @@ //TEST(compute, vulkan):COMPARE_COMPUTE(filecheck-buffer=BUF):-vk -compute -entry computeMain -allow-glsl //TEST(compute, vulkan):COMPARE_COMPUTE(filecheck-buffer=BUF):-vk -compute -entry computeMain -allow-glsl -emit-spirv-directly +//TEST(compute, vulkan):COMPARE_COMPUTE(filecheck-buffer=BUF):-wgpu -compute -entry computeMain -allow-glsl -xslang -DWGPU #version 430 //TEST_INPUT:ubuffer(data=[0], stride=4):out,name=outputBuffer @@ -24,15 +25,17 @@ void computeMain() { if (gl_GlobalInvocationID.x == 3) { outputBuffer.data[0] = true - && gl_NumSubgroups == 1 - && gl_SubgroupID == 0 //1 subgroup, 0 based indexing && gl_SubgroupSize == 32 && gl_SubgroupInvocationID == 3 +#if !defined(WGPU) + && gl_SubgroupID == 0 //1 subgroup, 0 based indexing + && gl_NumSubgroups == 1 && gl_SubgroupEqMask == uvec4(0b1000,0,0,0) && gl_SubgroupGeMask == uvec4(0xFFFFFFF8,0,0,0) && gl_SubgroupGtMask == uvec4(0xFFFFFFF0,0,0,0) && gl_SubgroupLeMask == uvec4(0b1111,0,0,0) && gl_SubgroupLtMask == uvec4(0b111,0,0,0) +#endif ; } // CHECK_GLSL: void main( diff --git a/tests/hlsl-intrinsic/wave-get-lane-index.slang b/tests/hlsl-intrinsic/wave-get-lane-index.slang index fb09022c23..e9b917442a 100644 --- a/tests/hlsl-intrinsic/wave-get-lane-index.slang +++ b/tests/hlsl-intrinsic/wave-get-lane-index.slang @@ -4,6 +4,7 @@ //TEST:COMPARE_COMPUTE_EX:-slang -compute -dx12 -use-dxil -profile cs_6_0 -shaderobj //TEST(vulkan):COMPARE_COMPUTE_EX:-vk -compute -shaderobj //TEST:COMPARE_COMPUTE_EX:-cuda -compute -shaderobj +//TEST:COMPARE_COMPUTE_EX:-wgpu -compute -shaderobj //TEST_INPUT:ubuffer(data=[0 0 0 0], stride=4):out,name outputBuffer RWStructuredBuffer<int> outputBuffer; diff --git a/tests/wgsl/implicit-system-values-dynamic-dispatch.slang b/tests/wgsl/implicit-system-values-dynamic-dispatch.slang new file mode 100644 index 0000000000..7a8c6a1e59 --- /dev/null +++ b/tests/wgsl/implicit-system-values-dynamic-dispatch.slang @@ -0,0 +1,68 @@ +// Test calling differentiable function through dynamic dispatch. + +//TEST(compute):COMPARE_COMPUTE(filecheck-buffer=BUF):-wgpu -compute -entry computeMain -output-using-type + +//TEST_INPUT:ubuffer(data=[0 0 0 0], stride=4):out,name=outputBuffer +RWStructuredBuffer<uint> outputBuffer; + +//TEST_INPUT: type_conformance Impl1:IInterface = 0 +//TEST_INPUT: type_conformance Impl2:IInterface = 1 +//TEST_INPUT: type_conformance Impl3:IInterface = 2 + +[anyValueSize(16)] +interface IInterface +{ + uint getLaneIndex(uint base); +} + +struct Impl1 : IInterface +{ + uint getLaneIndex(uint base) + { + return base; + } +} + +struct Impl2 : IInterface +{ + uint getLaneIndex(uint base) + { + return base * WaveGetLaneIndex() * 2; + } +} + +struct Impl3 : IInterface +{ + uint getLaneIndex(uint base) + { + return base + WaveGetLaneIndex(); + } +}; + +[numthreads(2, 1, 1)] +void computeMain(uint3 dispatchThreadID : SV_DispatchThreadID) +{ + const uint base = 5; + + if (dispatchThreadID.x == 0) + { + var obj = createDynamicObject<IInterface>(dispatchThreadID.x, 0); // Impl0 + outputBuffer[0] = obj.getLaneIndex(base); + + obj = createDynamicObject<IInterface>(dispatchThreadID.x + 1, 0); // Impl1 + outputBuffer[1] = obj.getLaneIndex(base); + } + else + { + var obj = createDynamicObject<IInterface>(dispatchThreadID.x, 0); // Impl1 + outputBuffer[2] = obj.getLaneIndex(base); + + obj = createDynamicObject<IInterface>(dispatchThreadID.x + 1, 0); // Impl2 + outputBuffer[3] = obj.getLaneIndex(base); + } + + // BUF: 5 + // BUF-NEXT: 0 + // BUF-NEXT: 10 + // BUF-NEXT: 6 +} diff --git a/tests/wgsl/implicit-system-values.slang b/tests/wgsl/implicit-system-values.slang new file mode 100644 index 0000000000..4d1f3e7658 --- /dev/null +++ b/tests/wgsl/implicit-system-values.slang @@ -0,0 +1,122 @@ +//TEST(compute):COMPARE_COMPUTE(filecheck-buffer=BUF):-wgpu -compute -entry computeMain + +//TEST_INPUT:ubuffer(data=[0], stride=4):out,name=outputBuffer +RWStructuredBuffer<uint> outputBuffer; + +interface IInterface +{ + uint getLaneIndex(); + uint getLaneIndex(uint base); +}; + +struct Impl1 : IInterface +{ + uint getLaneIndex() + { + return WaveGetLaneIndex(); + } + + uint getLaneIndex(uint base) + { + return base + WaveGetLaneIndex(); + } +}; + +struct Impl2 : IInterface +{ + uint getLaneIndex() + { + return 100; + } + + uint getLaneIndex(uint base) + { + return base - 1; + } +}; + +struct Impl3 : IInterface +{ + uint getLaneIndex(uint base = 1) + { + return base + WaveGetLaneIndex() + 1; + } +}; + +struct Impl4 : IInterface +{ + uint getLaneIndex() + { + return WaveGetLaneIndex() + 2; + } + + uint getLaneIndex(uint base) + { + return base * 2; + } +}; + + + +uint getLaneIndexGeneric<T>(T interface, uint base = 3) where T : IInterface +{ + return interface.getLaneIndex(base); +} + +struct MyStruct<T> where T : IInterface +{ + T interface; + uint getLaneIndex(uint base = 4) + { + return interface.getLaneIndex(base); + } +} + +[numthreads(1,1,1)] +void computeMain() +{ + Impl1 impl1; + Impl2 impl2; + Impl3 impl3; + Impl4 impl4; + + MyStruct<Impl1> s1; + MyStruct<Impl2> s2; + MyStruct<Impl3> s3; + MyStruct<Impl4> s4; + + // BUF: 1 + outputBuffer[0] = uint( + (0 == WaveGetLaneIndex()) + + // Interface implementations. + && (0 == impl1.getLaneIndex()) + && (100 == impl2.getLaneIndex()) + && (2 == impl3.getLaneIndex()) + && (2 == impl4.getLaneIndex()) + && (2 == impl1.getLaneIndex(2)) + && (1 == impl2.getLaneIndex(2)) + && (3 == impl3.getLaneIndex(2)) + && (12 == impl4.getLaneIndex(6)) + + // Interface as function generic parameter. + && (3 == getLaneIndexGeneric(impl1)) + && (2 == getLaneIndexGeneric(impl2)) + && (4 == getLaneIndexGeneric(impl3)) + && (6 == getLaneIndexGeneric(impl4)) + && (4 == getLaneIndexGeneric(impl1, 4)) + && (3 == getLaneIndexGeneric(impl2, 4)) + && (5 == getLaneIndexGeneric(impl3, 4)) + && (8 == getLaneIndexGeneric(impl4, 4)) + + // Interface as struct generic member. + && (5 == s1.getLaneIndex(5)) + && (4 == s2.getLaneIndex(5)) + && (6 == s3.getLaneIndex(5)) + && (10 == s4.getLaneIndex(5)) + && (4 == s1.getLaneIndex()) + && (3 == s2.getLaneIndex()) + && (5 == s3.getLaneIndex()) + && (8 == s4.getLaneIndex()) + ); +}