For the complete documentation index, see llms.txt. Markdown versions of all pages are available by appending .md to any URL (e.g. /get-started.md).
Mojo struct
HopperMatmulSM90Kernel
struct HopperMatmulSM90Kernel[a_type: DType, b_type: DType, c_type: DType, a_layout: TensorLayout, b_layout: TensorLayout, c_layout: TensorLayout, c_smem_rows: Int, c_smem_cols: Int, block_tile_shape: Tuple[Int, Int, Int], wgmma_shape: Tuple[Int, Int, Int], cluster_shape: Tuple[Int, Int, Int], num_pipeline_stages: Int, num_threads: Int = Int(128), transpose_b: Bool = True, a_swizzle: TensorMapSwizzle = TensorMapSwizzle.SWIZZLE_128B, b_swizzle: TensorMapSwizzle = TensorMapSwizzle.SWIZZLE_128B, c_swizzle: TensorMapSwizzle = TensorMapSwizzle.SWIZZLE_NONE, partitioned_multicast: Bool = False, use_tma_store: Bool = False, promotion_frequency: Int = Int(1), hilbert_swizzle: Bool = False, k_group_size: Int = Int(1), swapAB: Bool = False, a_engine: TensorEngine = DefaultEngine, b_engine: TensorEngine = DefaultEngine, c_engine: TensorEngine = DefaultEngine, a_offsets_engine: TensorEngine = DefaultEngine, expert_ids_engine: TensorEngine = DefaultEngine]
Hopper SM90 GEMM for NVIDIA H100 GPUs.
Uses TMA loads, WGMMA tensor-core MMA, multi-stage pipelining, and a producer-consumer warp-group layout.
Parameters
- a_type (
DType): Data type of the A input matrix. - b_type (
DType): Data type of the B input matrix. - c_type (
DType): Data type of the C output matrix. - a_layout (
TensorLayout): Memory layout of the A matrix. - b_layout (
TensorLayout): Memory layout of the B matrix. - c_layout (
TensorLayout): Memory layout of the C matrix. - c_smem_rows (
Int): Row count of the output tile's shared memory layout. - c_smem_cols (
Int): Column count of the output tile's shared memory layout. - block_tile_shape (
Tuple[Int, Int, Int]): Tile dimensions[M, N, K]processed by each thread block. - wgmma_shape (
Tuple[Int, Int, Int]): Dimensions for each WGMMA instruction[M, N, K]. - cluster_shape (
Tuple[Int, Int, Int]): Thread block cluster dimensions for distributed shared memory. - num_pipeline_stages (
Int): Number of stages in the software pipeline (3-7 in most configs). - num_threads (
Int): Number of threads per block (must be a multiple of 128). - transpose_b (
Bool): Whether the B matrix is transposed (required to beTrue). - a_swizzle (
TensorMapSwizzle): Memory swizzling for bank-conflict-free A tile access. - b_swizzle (
TensorMapSwizzle): Memory swizzling for bank-conflict-free B tile access. - c_swizzle (
TensorMapSwizzle): Swizzling mode for output writes. - partitioned_multicast (
Bool): Whether partitioned multicast is enabled for large tiles. - use_tma_store (
Bool): Whether TMA is used for storing output (versus regular stores). - promotion_frequency (
Int): How often FP8 accumulation is promoted to higher precision. - hilbert_swizzle (
Bool): Whether Hilbert-curve thread block scheduling is used. - k_group_size (
Int): Number of K-dimension tiles loaded and consumed per pipeline stage; bothnum_pipeline_stagesand the total K extent must be multiples of this value (defaults to 1). - swapAB (
Bool): Whether to swap the A and B operand roles in the output writer for the small-M strategy, transposing the tile and block coordinate mapping (defaults toFalse). - a_engine (
TensorEngine): Engine of the A operand (defaults toDefaultEngine). - b_engine (
TensorEngine): Engine of the B operand (defaults toDefaultEngine). - c_engine (
TensorEngine): Engine of the C output (defaults toDefaultEngine). - a_offsets_engine (
TensorEngine): Engine of thea_offsetstensor (defaults toDefaultEngine). - expert_ids_engine (
TensorEngine): Engine of theexpert_idstensor (defaults toDefaultEngine).
Implemented traits
comptime members
accum_type
comptime accum_type = get_accum_type[a_type]()
AccumRegTile
comptime AccumRegTile = TileTensor[HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].accum_type, Layout[TypeList[ComptimeInt[(HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].num_m_mmas * HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].num_n_mmas)], ComptimeInt[HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]
AccumRegTileLayout
comptime AccumRegTileLayout = row_major[(HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].num_m_mmas * HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].num_n_mmas), HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].c_frag_size]()
adjusted_num_pipeline_stages
comptime adjusted_num_pipeline_stages = (num_pipeline_stages // k_group_size)
BK
comptime BK = (load_from_mem block_tile_shape.__getitem_param__[Int(2)]())
BM
comptime BM = (load_from_mem block_tile_shape.__getitem_param__[Int(0)]())
BN
comptime BN = (load_from_mem block_tile_shape.__getitem_param__[Int(1)]())
c_frag_size
comptime c_frag_size = ((SIMD((load_from_mem wgmma_shape.__getitem_param__[Int(0)]())) * SIMD((load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))
cluster_size
comptime cluster_size = SIMD((((load_from_mem cluster_shape.__getitem_param__[Int(0)]()) * (load_from_mem cluster_shape.__getitem_param__[Int(1)]())) * (load_from_mem cluster_shape.__getitem_param__[Int(2)]())))
num_consumer
comptime num_consumer = ((num_threads // Int(128)) - Int(1))
num_consumer_threads
comptime num_consumer_threads = (HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].num_consumer * Int(128))
num_m_mmas
comptime num_m_mmas = ((HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].BM // SIMD((load_from_mem wgmma_shape.__getitem_param__[Int(0)]()))) // HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].num_consumer)
num_n_mmas
comptime num_n_mmas = (HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].BN // SIMD((load_from_mem wgmma_shape.__getitem_param__[Int(1)]())))
SMem
comptime SMem = HopperMatmulSM90Kernel_SMem[a_type, b_type, c_type, HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].BM, HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].BN, HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].BK, c_smem_rows, c_smem_cols, num_pipeline_stages, k_group_size]
TMABarrier
comptime TMABarrier = TMABarrierHandler[(Int((add (mul Layout(Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt())), Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt()))).product(), size_of[a_type](), num_pipeline_stages), (mul Layout(Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt())), Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt()))).product(), size_of[b_type](), num_pipeline_stages))) // HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].adjusted_num_pipeline_stages)]
WgmmaOp
comptime WgmmaOp = TensorCoreAsync[HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].accum_type, a_type, b_type, wgmma_shape, a_swizzle, b_swizzle, transpose_b]
Methods
validate_constraints
static def validate_constraints()
Validate common constraints for all kernel variants.
pipeline_init
static def pipeline_init()
Initialize pipeline synchronization barriers.
This function ensures that all pipeline initialization (barriers, shared memory) is visible to all thread blocks in the cluster before proceeding. This is critical for correct producer-consumer synchronization.
For multi-cluster configurations, uses fence and cluster sync. For single block, uses a simple barrier.
finalize_kernel
static def finalize_kernel()
Common finalization for all kernel variants.
multicast_mask
static def multicast_mask(rank_m: Int, rank_n: Int) -> Tuple[Int32, Int32]
Returns:
common_kernel_init
static def common_kernel_init() -> Tuple[Int, Int, Int, Int, Int, Bool]
Common initialization for all kernel variants.
Returns:
Tuple[Int, Int, Int, Int, Int, Bool]: Tuple of (warp_group_idx, warp_group_thread_idx,
rank_m, rank_n, warp_id, lane_predicate).
setup_producer
static def setup_producer() -> Int
Setup producer warp group by deallocating registers.
Returns:
Int: Number of registers deallocated.
allocate_accum_reg_tile
static def allocate_accum_reg_tile() -> Self.AccumRegTile
Allocates one WGMMA accumulator tile in registers.
Returns:
Self.AccumRegTile
setup_consumer
static def setup_consumer(warp_group_idx: Int) -> Tuple[Int, TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL], TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]]
Setup consumer warp group.
Returns:
consumer_arrive_empty_barriers
static def consumer_arrive_empty_barriers(warp_group_thread_idx: Int, mut pipeline: ProducerConsumerPipeline[Self.adjusted_num_pipeline_stages])
Signal initial empty barrier arrival for all pipeline stages.
Must be called by consumer warp groups before the main loop so the producer knows it can start filling stages.
Args:
- warp_group_thread_idx (
Int): Thread index within the warp group, used to select which threads signal the barrier. - pipeline (
ProducerConsumerPipeline[Self.adjusted_num_pipeline_stages]): Producer-consumer pipeline whose empty barriers are signaled (modified in place).
get_block_swizzle
static def get_block_swizzle(lut_ptr: OptionalReg[Pointer[UInt32, MutAnyOrigin]] = None) -> IndexList[Int(2), element_type=.uint32]
Calculate block swizzle for better L2 cache locality.
Args:
- lut_ptr (
OptionalReg[Pointer[UInt32, MutAnyOrigin]]): Lookup table for Hilbert curve block scheduling (optional).
Returns:
IndexList[Int(2), element_type=.uint32]: Swizzled block indices.
consumer_output
static def consumer_output[EpilogueFnType: ElementwiseEpilogueFn = inflated$def[comptime[dtype: DType, width: SIMDLength, *, alignment: Int] #kgen.func.symbol<@linalg::@utils::@"_no_epilogue_body[::DType,::SIMDLength,::SIMD[DType.int, 1],KGENParamList[::Intable & ::Defaultable & ::RegisterPassable & ::TrivialRegisterPassable & ::Writable & ::AnyType & ::Copyable & ::ImplicitlyCopyable & ::Deinitable & ::Movable & ::CoordLike],::TypeList[::Intable & ::Defaultable & ::RegisterPassable & ::TrivialRegisterPassable & ::Writable & ::AnyType & ::Copyable & ::ImplicitlyCopyable & ::Deinitable & ::Movable & ::CoordLike, $3]](::Coord[$3, $4],::SIMD[$0, $1])"<:!lit.struct<@std::@builtin::@dtype::@DType> dtype, :!lit.struct<@std::@builtin::@simd_length::@SIMDLength> width, :!lit.struct<@std::@simd::@SIMD<:!lit.struct<@std::@builtin::@dtype::@DType> {:dtype index}, :!lit.struct<@std::@builtin::@simd_length::@SIMDLength> {1}>> alignment, :param_list<trait<@std::@builtin::@int::@Intable, @std::@builtin::@value::@Defaultable, @std::@builtin::@value::@RegisterPassable, @std::@builtin::@value::@TrivialRegisterPassable, @std::@format::@__init__::@Writable, @std::@traits::@anytype::@AnyType, @std::@traits::@copyable::@Copyable, @std::@traits::@copyable::@ImplicitlyCopyable, @std::@traits::@deinitable::@Deinitable, @std::@traits::@movable::@Movable, @std::@utils::@coord::@CoordLike>> *?, :!lit.struct<@std::@builtin::@variadics::@TypeList<:meta<!lit.trait<@std::@traits::@anytype::@AnyType>> trait<@std::@builtin::@int::@Intable, @std::@builtin::@value::@Defaultable, @std::@builtin::@value::@RegisterPassable, @std::@builtin::@value::@TrivialRegisterPassable, @std::@format::@__init__::@Writable, @std::@traits::@anytype::@AnyType, @std::@traits::@copyable::@Copyable, @std::@traits::@copyable::@ImplicitlyCopyable, @std::@traits::@deinitable::@Deinitable, @std::@traits::@movable::@Movable, @std::@utils::@coord::@CoordLike>, :param_list<trait<@std::@builtin::@int::@Intable, @std::@builtin::@value::@Defaultable, @std::@builtin::@value::@RegisterPassable, @std::@builtin::@value::@TrivialRegisterPassable, @std::@format::@__init__::@Writable, @std::@traits::@anytype::@AnyType, @std::@traits::@copyable::@Copyable, @std::@traits::@copyable::@ImplicitlyCopyable, @std::@traits::@deinitable::@Deinitable, @std::@traits::@movable::@Movable, @std::@utils::@coord::@CoordLike>> *?>> *?>>], ComputeFnType: ElementwiseComputeFn = inflated$def[comptime[dtype: DType, width: SIMDLength, *, alignment: Int] #kgen.func.symbol<@linalg::@utils::@"identity_compute_fn[::DType,::SIMDLength,::SIMD[DType.int, 1],KGENParamList[::Intable & ::Defaultable & ::RegisterPassable & ::TrivialRegisterPassable & ::Writable & ::AnyType & ::Copyable & ::ImplicitlyCopyable & ::Deinitable & ::Movable & ::CoordLike],::TypeList[::Intable & ::Defaultable & ::RegisterPassable & ::TrivialRegisterPassable & ::Writable & ::AnyType & ::Copyable & ::ImplicitlyCopyable & ::Deinitable & ::Movable & ::CoordLike, $3]](::Coord[$3, $4],::SIMD[$0, $1])"<:!lit.struct<@std::@builtin::@dtype::@DType> dtype, :!lit.struct<@std::@builtin::@simd_length::@SIMDLength> width, :!lit.struct<@std::@simd::@SIMD<:!lit.struct<@std::@builtin::@dtype::@DType> {:dtype index}, :!lit.struct<@std::@builtin::@simd_length::@SIMDLength> {1}>> alignment, :param_list<trait<@std::@builtin::@int::@Intable, @std::@builtin::@value::@Defaultable, @std::@builtin::@value::@RegisterPassable, @std::@builtin::@value::@TrivialRegisterPassable, @std::@format::@__init__::@Writable, @std::@traits::@anytype::@AnyType, @std::@traits::@copyable::@Copyable, @std::@traits::@copyable::@ImplicitlyCopyable, @std::@traits::@deinitable::@Deinitable, @std::@traits::@movable::@Movable, @std::@utils::@coord::@CoordLike>> *?, :!lit.struct<@std::@builtin::@variadics::@TypeList<:meta<!lit.trait<@std::@traits::@anytype::@AnyType>> trait<@std::@builtin::@int::@Intable, @std::@builtin::@value::@Defaultable, @std::@builtin::@value::@RegisterPassable, @std::@builtin::@value::@TrivialRegisterPassable, @std::@format::@__init__::@Writable, @std::@traits::@anytype::@AnyType, @std::@traits::@copyable::@Copyable, @std::@traits::@copyable::@ImplicitlyCopyable, @std::@traits::@deinitable::@Deinitable, @std::@traits::@movable::@Movable, @std::@utils::@coord::@CoordLike>, :param_list<trait<@std::@builtin::@int::@Intable, @std::@builtin::@value::@Defaultable, @std::@builtin::@value::@RegisterPassable, @std::@builtin::@value::@TrivialRegisterPassable, @std::@format::@__init__::@Writable, @std::@traits::@anytype::@AnyType, @std::@traits::@copyable::@Copyable, @std::@traits::@copyable::@ImplicitlyCopyable, @std::@traits::@deinitable::@Deinitable, @std::@traits::@movable::@Movable, @std::@utils::@coord::@CoordLike>> *?>> *?>>], //, *, has_epilogue_fn: Bool = False, has_compute_fn: Bool = False](c_tma_op: TMATensorTile[c_type], c: TileTensor[c_type, Engine=c.Engine, linear_idx_type=c.linear_idx_type], c_tile: TileTensor[c_type, Layout[TypeList[ComptimeInt[c_smem_rows], ComptimeInt[c_smem_cols]](), TypeList[ComptimeInt[c_smem_cols], ComptimeInt[Int(1)]]()], MutAnyOrigin, address_space=AddressSpace.SHARED], output_reg_tile: TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL], warp_group_thread_idx: Int, local_warp_group_idx: Int, local_thread_idx: Int, block_y: Int, block_x: Int, epilogue_fn: EpilogueFnType = no_epilogue_fn, compute_fn: ComputeFnType = no_compute_fn)
Handle consumer output by writing GEMM results to global memory.
Parameters:
- EpilogueFnType (
ElementwiseEpilogueFn): Type ofepilogue_fn(inferred). - ComputeFnType (
ElementwiseComputeFn): Type ofcompute_fn(inferred). - has_epilogue_fn (
Bool): Whetherepilogue_fnstores the output. - has_compute_fn (
Bool): Whethercompute_fnmaps each output before it is stored.
Args:
- c_tma_op (
TMATensorTile[c_type]): TMA descriptor for the output matrix C, used for TMA stores. - c (
TileTensor[c_type, Engine=c.Engine, linear_idx_type=c.linear_idx_type]): Writable output matrix C tile tensor. - c_tile (
TileTensor[c_type, Layout[TypeList[ComptimeInt[c_smem_rows], ComptimeInt[c_smem_cols]](), TypeList[ComptimeInt[c_smem_cols], ComptimeInt[Int(1)]]()], MutAnyOrigin, address_space=AddressSpace.SHARED]): Shared memory tile staging the output before the global write. - output_reg_tile (
TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]): Register tile holding the accumulated GEMM result to write. - warp_group_thread_idx (
Int): Thread index within the warp group. - local_warp_group_idx (
Int): Index of this consumer warp group (0-based). - local_thread_idx (
Int): Thread index within the consumer warp group. - block_y (
Int): Block-level M coordinate (row) of the output tile. - block_x (
Int): Block-level N coordinate (column) of the output tile. - epilogue_fn (
EpilogueFnType): Stores each output element at its coordinates incwhenhas_epilogue_fnis set. - compute_fn (
ComputeFnType): Maps each output element at its coordinates incwhenhas_compute_fnis set.
build_tma_loaders
static def build_tma_loaders[a_tile_shape: Coord[*a_tile_shape.element_types], b_tile_shape: Coord[*b_tile_shape.element_types], a_desc_shape: Coord[*a_desc_shape.element_types], b_desc_shape: Coord[*b_desc_shape.element_types], //](a_tma_op: TMATensorTile[a_type, a_tile_shape, a_desc_shape], b_tma_op: TMATensorTile[b_type, b_tile_shape, b_desc_shape], rank_m: Int, rank_n: Int) -> Tuple[TileLoaderTMA[origin_of(a_tma_op), a_type, a_tile_shape, a_desc_shape, BK=Self.BK, cluster_size=SIMD((load_from_mem cluster_shape.__getitem_param__[Int(0)]())), use_partitioned_multicast=partitioned_multicast], TileLoaderTMA[origin_of(b_tma_op), b_type, b_tile_shape, b_desc_shape, BK=Self.BK, cluster_size=SIMD((load_from_mem cluster_shape.__getitem_param__[Int(1)]())), use_partitioned_multicast=partitioned_multicast]]
Returns:
build_cpasync_loaders
static def build_cpasync_loaders[k_align: Int, vector_size: Int = (k_align // size_of[a_type]()), num_threads_per_row: Int = (HopperMatmulSM90Kernel[a_type, b_type, c_type, a_layout, b_layout, c_layout, c_smem_rows, c_smem_cols, block_tile_shape, wgmma_shape, cluster_shape, num_pipeline_stages, num_threads, transpose_b, a_swizzle, b_swizzle, c_swizzle, partitioned_multicast, use_tma_store, promotion_frequency, hilbert_swizzle, k_group_size, swapAB, a_engine, b_engine, c_engine, a_offsets_engine, expert_ids_engine].BK // vector_size), thread_layout: Layout[thread_layout.shape_types, thread_layout.stride_types] = row_major[(_resolve_warpgroup_size() // num_threads_per_row), num_threads_per_row]()](a: TileTensor[a_type, a_layout, ImmutAnyOrigin, Engine=a_engine], b: TileTensor[b_type, b_layout, ImmutAnyOrigin, Engine=b_engine]) -> Tuple[TileLoaderCPAsync[a_type, a_layout, thread_layout, a_swizzle, vector_size, a_engine], TileLoaderCPAsync[b_type, b_layout, thread_layout, b_swizzle, vector_size, b_engine]]
Returns:
producer_main_loop_pipeline
static def producer_main_loop_pipeline[a_loader_type: TileLoader, b_loader_type: TileLoader, barrier_handler_type: BarrierHandler, //, num_k_iters: Int](m_coord: Int, n_coord: Int, k_coord: Int, a_loader: a_loader_type, b_loader: b_loader_type, barrier_handler: barrier_handler_type, mut pipeline: ProducerConsumerPipeline[Self.adjusted_num_pipeline_stages], a_tiles: SMemTileArrayWithLayout[a_type, Layout(Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt())), Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt()))), num_pipeline_stages], b_tiles: SMemTileArrayWithLayout[b_type, Layout(Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt())), Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt()))), num_pipeline_stages])
run
static def run[a_tile_shape: Coord[*a_tile_shape.element_types], b_tile_shape: Coord[*b_tile_shape.element_types], c_tile_shape: Coord[*c_tile_shape.element_types], a_desc_shape: Coord[*a_desc_shape.element_types], b_desc_shape: Coord[*b_desc_shape.element_types], c_desc_shape: Coord[*c_desc_shape.element_types], a_tensor_layout: TensorLayout, b_tensor_layout: TensorLayout, c_tensor_layout: TensorLayout, EpilogueFnType: ElementwiseEpilogueFn, has_epilogue_fn: Bool, ComputeFnType: ElementwiseComputeFn, has_compute_fn: Bool](a_tma_op: TMATensorTile[a_type, a_tile_shape, a_desc_shape], b_tma_op: TMATensorTile[b_type, b_tile_shape, b_desc_shape], c_tma_op: TMATensorTile[c_type, c_tile_shape, c_desc_shape], a: TileTensor[a_type, a_tensor_layout, ImmutAnyOrigin, Engine=a_engine], b: TileTensor[b_type, b_tensor_layout, ImmutAnyOrigin, Engine=b_engine], c: TileTensor[c_type, c_tensor_layout, MutAnyOrigin, Engine=c_engine], lut_ptr: MutPointer[UInt32, MutAnyOrigin], epilogue_fn: EpilogueFnType, compute_fn: ComputeFnType)
Main kernel entry point for matrix multiplication.
This kernel implements a producer-consumer pattern where:
- One warp group (producer) loads tiles from global memory using TMA
- Multiple warp groups (consumers) perform matrix multiplication using tensor cores
The kernel uses software pipelining to overlap memory transfers with computation, achieving high throughput on Hopper GPUs.
Parameters:
- a_tile_shape (
Coord[*element_types]): Shape of each A tile loaded by TMA. - b_tile_shape (
Coord[*element_types]): Shape of each B tile loaded by TMA. - c_tile_shape (
Coord[*element_types]): Shape of each C tile stored by TMA. - a_desc_shape (
Coord[*element_types]): Full shape of matrix A as described by the TMA descriptor. - b_desc_shape (
Coord[*element_types]): Full shape of matrix B as described by the TMA descriptor. - c_desc_shape (
Coord[*element_types]): Full shape of matrix C as described by the TMA descriptor. - a_tensor_layout (
TensorLayout): Memory layout of input matrix A. - b_tensor_layout (
TensorLayout): Memory layout of input matrix B. - c_tensor_layout (
TensorLayout): Memory layout of output matrix C. - EpilogueFnType (
ElementwiseEpilogueFn): Type ofepilogue_fn. - has_epilogue_fn (
Bool): Whetherepilogue_fnstores the output. Launch withepilogue_fnas the firsthost_argselement. - ComputeFnType (
ElementwiseComputeFn): Type ofcompute_fn. - has_compute_fn (
Bool): Whethercompute_fnmaps each output before it is stored. Launch withcompute_fnas the secondhost_argselement.
Args:
- a_tma_op (
TMATensorTile[a_type, a_tile_shape, a_desc_shape]): TMA descriptor for matrix A. - b_tma_op (
TMATensorTile[b_type, b_tile_shape, b_desc_shape]): TMA descriptor for matrix B. - c_tma_op (
TMATensorTile[c_type, c_tile_shape, c_desc_shape]): TMA descriptor for matrix C. - a (
TileTensor[a_type, a_tensor_layout, ImmutAnyOrigin, Engine=a_engine]): Input matrix A. - b (
TileTensor[b_type, b_tensor_layout, ImmutAnyOrigin, Engine=b_engine]): Input matrix B. - c (
TileTensor[c_type, c_tensor_layout, MutAnyOrigin, Engine=c_engine]): Output matrix C. - lut_ptr (
MutPointer[UInt32, MutAnyOrigin]): Lookup table for Hilbert curve block scheduling (optional). - epilogue_fn (
EpilogueFnType): Stores each output element at its(row, col)incwhenhas_epilogue_fnis set. - compute_fn (
ComputeFnType): Maps each output element at its(row, col)incwhenhas_compute_fnis set.
run_splitk
static def run_splitk[a_tile_shape: Coord[*a_tile_shape.element_types], b_tile_shape: Coord[*b_tile_shape.element_types], c_tile_shape: Coord[*c_tile_shape.element_types], a_desc_shape: Coord[*a_desc_shape.element_types], b_desc_shape: Coord[*b_desc_shape.element_types], c_desc_shape: Coord[*c_desc_shape.element_types], splits: Int, raster_order: RasterOrder, c_tensor_layout: TensorLayout, EpilogueFnType: ElementwiseEpilogueFn, has_epilogue_fn: Bool, ComputeFnType: ElementwiseComputeFn, has_compute_fn: Bool](a_tma_op: TMATensorTile[a_type, a_tile_shape, a_desc_shape], b_tma_op: TMATensorTile[b_type, b_tile_shape, b_desc_shape], c_tma_op: TMATensorTile[c_type, c_tile_shape, c_desc_shape], c: TileTensor[c_type, c_tensor_layout, MutAnyOrigin, Engine=c_engine], workspace_ptr: MutPointer[Scalar[Self.accum_type], MutAnyOrigin], locks_ptr: MutPointer[UInt8, MutAnyOrigin], problem_shape: IndexList[Int(3)], epilogue_fn: EpilogueFnType, compute_fn: ComputeFnType)
Split-K variant of the kernel for better load balancing on small problems.
Parameters:
- a_tile_shape (
Coord[*element_types]): Shape of each A tile loaded by TMA. - b_tile_shape (
Coord[*element_types]): Shape of each B tile loaded by TMA. - c_tile_shape (
Coord[*element_types]): Shape of each C tile stored by TMA. - a_desc_shape (
Coord[*element_types]): Full shape of matrix A as described by the TMA descriptor. - b_desc_shape (
Coord[*element_types]): Full shape of matrix B as described by the TMA descriptor. - c_desc_shape (
Coord[*element_types]): Full shape of matrix C as described by the TMA descriptor. - splits (
Int): Number of equal chunks the K dimension is divided into for parallel reduction. Each block processes one chunk per output tile. - raster_order (
RasterOrder): Tile rasterization order used by the split-K scheduler to assign output tiles to blocks. - c_tensor_layout (
TensorLayout): Memory layout of output matrix C. - EpilogueFnType (
ElementwiseEpilogueFn): Type ofepilogue_fn. - has_epilogue_fn (
Bool): Whetherepilogue_fnstores the output. Launch withepilogue_fnas the firsthost_argselement. - ComputeFnType (
ElementwiseComputeFn): Type ofcompute_fn. - has_compute_fn (
Bool): Whethercompute_fnmaps each output before it is stored. Launch withcompute_fnas the secondhost_argselement.
Args:
- a_tma_op (
TMATensorTile[a_type, a_tile_shape, a_desc_shape]): TMA descriptor for matrix A. - b_tma_op (
TMATensorTile[b_type, b_tile_shape, b_desc_shape]): TMA descriptor for matrix B. - c_tma_op (
TMATensorTile[c_type, c_tile_shape, c_desc_shape]): TMA descriptor for matrix C. - c (
TileTensor[c_type, c_tensor_layout, MutAnyOrigin, Engine=c_engine]): Output matrix C. - workspace_ptr (
MutPointer[Scalar[Self.accum_type], MutAnyOrigin]): Pointer to the reduction workspace storing partial accumulations from each split. - locks_ptr (
MutPointer[UInt8, MutAnyOrigin]): Pointer to the lock array coordinating split-K synchronization across blocks. - problem_shape (
IndexList[Int(3)]): Full GEMM problem dimensions[M, N, K]. - epilogue_fn (
EpilogueFnType): Stores each output element at its(row, col)incwhenhas_epilogue_fnis set. - compute_fn (
ComputeFnType): Maps each output element at its(row, col)incwhenhas_compute_fnis set.
run_grouped
static def run_grouped[a_tile_shape: Coord[*a_tile_shape.element_types], b_tile_shape: Coord[*b_tile_shape.element_types], c_tile_shape: Coord[*c_tile_shape.element_types], a_desc_shape: Coord[*a_desc_shape.element_types], b_desc_shape: Coord[*b_desc_shape.element_types], c_desc_shape: Coord[*c_desc_shape.element_types], AOffsetsLayout: TensorLayout, ExpertIdsLayout: TensorLayout, c_tensor_layout: TensorLayout, EpilogueFnType: ElementwiseEpilogueFn, has_epilogue_fn: Bool, ComputeFnType: ElementwiseComputeFn, has_compute_fn: Bool](a_tma_op: TMATensorTile[a_type, a_tile_shape, a_desc_shape], b_tma_op: TMATensorTile[b_type, b_tile_shape, b_desc_shape], c_tma_op: TMATensorTile[c_type, c_tile_shape, c_desc_shape], a_offsets: TileTensor[.uint32, AOffsetsLayout, ImmUnsafeAnyOrigin, Engine=a_offsets_engine], expert_ids: TileTensor[.int32, ExpertIdsLayout, ImmUnsafeAnyOrigin, Engine=expert_ids_engine], c: TileTensor[c_type, c_tensor_layout, MutAnyOrigin, Engine=c_engine], epilogue_fn: EpilogueFnType, compute_fn: ComputeFnType)
Grouped matmul variant for MoE (Mixture of Experts) models.
This variant handles multiple experts where each expert processes a subset of tokens. The a_offsets array indicates token boundaries for each expert.
Parameters:
- a_tile_shape (
Coord[*element_types]): Shape of each A tile loaded by TMA. - b_tile_shape (
Coord[*element_types]): Shape of each B tile loaded by TMA. - c_tile_shape (
Coord[*element_types]): Shape of each C tile stored by TMA. - a_desc_shape (
Coord[*element_types]): Full shape of matrix A as described by the TMA descriptor. - b_desc_shape (
Coord[*element_types]): Full shape of matrix B as described by the TMA descriptor. - c_desc_shape (
Coord[*element_types]): Full shape of matrix C as described by the TMA descriptor. - AOffsetsLayout (
TensorLayout): Memory layout of thea_offsetstensor. - ExpertIdsLayout (
TensorLayout): Memory layout of theexpert_idstensor. - c_tensor_layout (
TensorLayout): Memory layout of output matrix C. - EpilogueFnType (
ElementwiseEpilogueFn): Type ofepilogue_fn. - has_epilogue_fn (
Bool): Whetherepilogue_fnstores the output. Launch withepilogue_fnas the firsthost_argselement. - ComputeFnType (
ElementwiseComputeFn): Type ofcompute_fn. - has_compute_fn (
Bool): Whethercompute_fnmaps each output before it is stored. Launch withcompute_fnas the secondhost_argselement.
Args:
- a_tma_op (
TMATensorTile[a_type, a_tile_shape, a_desc_shape]): TMA descriptor for matrix A. - b_tma_op (
TMATensorTile[b_type, b_tile_shape, b_desc_shape]): TMA descriptor for matrix B. - c_tma_op (
TMATensorTile[c_type, c_tile_shape, c_desc_shape]): TMA descriptor for matrix C. - a_offsets (
TileTensor[.uint32, AOffsetsLayout, ImmUnsafeAnyOrigin, Engine=a_offsets_engine]): Starting row offsets into matrix A for each expert. The token count for expertiis `a_offsets[i+1]- a_offsets[i]
, indexed byblock_idx.z`.
- a_offsets[i]
- expert_ids (
TileTensor[.int32, ExpertIdsLayout, ImmUnsafeAnyOrigin, Engine=expert_ids_engine]): Expert index selected by each block.expert_ids[block_idx.z]picks the row block in B for this block, and -1 marks an inactive block whose output is zeroed. - c (
TileTensor[c_type, c_tensor_layout, MutAnyOrigin, Engine=c_engine]): Output matrix C. - epilogue_fn (
EpilogueFnType): Stores each output element at its(row, col)incwhenhas_epilogue_fnis set. - compute_fn (
ComputeFnType): Maps each output element at its(row, col)incwhenhas_compute_fnis set.
consumer_main_loop_pipeline
static def consumer_main_loop_pipeline[num_k_iters: Int](wgmma_op: TensorCoreAsync[Self.accum_type, a_type, b_type, wgmma_shape, a_swizzle, b_swizzle, transpose_b], local_warp_group_idx: Int, final_c_reg_tile: TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL], c_reg_tile: TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL], mut pipeline: ProducerConsumerPipeline[Self.adjusted_num_pipeline_stages], a_tiles: SMemTileArrayWithLayout[a_type, Layout(Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt())), Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt()))), num_pipeline_stages], b_tiles: SMemTileArrayWithLayout[b_type, Layout(Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt())), Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt()))), num_pipeline_stages], warp_group_thread_idx: Int)
Pipeline-based consumer loop using ProducerConsumerPipeline.
This is an alternative implementation of consumer_main_loop that uses the SM100 ProducerConsumerPipeline for synchronization instead of RingBuffer.
Parameters:
- num_k_iters (
Int): Number of K-dimension tiles the consumer processes in this loop.
Args:
- wgmma_op (
TensorCoreAsync[Self.accum_type, a_type, b_type, wgmma_shape, a_swizzle, b_swizzle, transpose_b]): Tensor core operator for matrix multiplication. - local_warp_group_idx (
Int): Index of this consumer warp group (0-based). - final_c_reg_tile (
TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]): Final accumulation register tile (for FP8 promotion). - c_reg_tile (
TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]): Working accumulation register tile. - pipeline (
ProducerConsumerPipeline[Self.adjusted_num_pipeline_stages]): ProducerConsumerPipeline for synchronized tile access. - a_tiles (
SMemTileArrayWithLayout[a_type, Layout(Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt())), Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt()))), num_pipeline_stages]): Tile array for A matrix in shared memory. - b_tiles (
SMemTileArrayWithLayout[b_type, Layout(Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt())), Coord(Coord(ComptimeInt(), ComptimeInt()), Coord(ComptimeInt(), ComptimeInt()))), num_pipeline_stages]): Tile array for B matrix in shared memory. - warp_group_thread_idx (
Int): Thread index within the warp group.
promote_to_cuda_cores
static def promote_to_cuda_cores(c_reg_tile: TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL], final_c_reg_tile: TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL])
Promote FP8 accumulation to higher precision using CUDA cores.
When using FP8 data types, tensor cores accumulate in limited precision. To maintain accuracy over many accumulations, we periodically add the intermediate results to a higher-precision accumulator using CUDA cores.
This technique is commonly used in production libraries like cuBLAS to achieve both high performance (from FP8 tensor cores) and good accuracy.
Args:
- c_reg_tile (
TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]): Current accumulation from tensor cores. - final_c_reg_tile (
TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]): Higher-precision accumulator (updated in place).
wgmma
static def wgmma(wgmma_op: TensorCoreAsync[Self.accum_type, a_type, b_type, wgmma_shape, a_swizzle, b_swizzle, transpose_b], local_warp_group_idx: Int, a_tile: TileTensor[a_type, Layout[TypeList[Coord[ComptimeInt[Int(8)], ComptimeInt[(Self.BM // Int(8))]], Coord[ComptimeInt[(Int(128) // size_of[a_type]())], ComptimeInt[(Int((mul size_of[a_type](), Self.BK)) // Int(128))]]](), TypeList[Coord[ComptimeInt[(Int(128) // size_of[a_type]())], ComptimeInt[(Int(8) * (Int(128) // size_of[a_type]()))]], Coord[ComptimeInt[Int(1)], ComptimeInt[Int(0) if identical((Int((mul size_of[a_type](), Self.BK)) // Int(128)), 1) else Int((mul (Int(128) // size_of[a_type]()), Self.BM))]]]()], MutAnyOrigin, address_space=AddressSpace.SHARED], b_tile: TileTensor[b_type, Layout[TypeList[Coord[ComptimeInt[Int(8)], ComptimeInt[(Self.BN // Int(8))]], Coord[ComptimeInt[(Int(128) // size_of[b_type]())], ComptimeInt[(Int((mul size_of[b_type](), Self.BK)) // Int(128))]]](), TypeList[Coord[ComptimeInt[(Int(128) // size_of[b_type]())], ComptimeInt[(Int(8) * (Int(128) // size_of[b_type]()))]], Coord[ComptimeInt[Int(1)], ComptimeInt[Int(0) if identical((Int((mul size_of[b_type](), Self.BK)) // Int(128)), 1) else Int((mul (Int(128) // size_of[b_type]()), Self.BN))]]]()], MutAnyOrigin, address_space=AddressSpace.SHARED], c_reg_tile: TileTensor[Self.accum_type, Layout[TypeList[ComptimeInt[(Self.num_m_mmas * Self.num_n_mmas)], ComptimeInt[Self.c_frag_size]](), TypeList[ComptimeInt[(Int((mul (load_from_mem wgmma_shape.__getitem_param__[Int(0)]()), (load_from_mem wgmma_shape.__getitem_param__[Int(1)]()))) // Int(128))], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL])