For the complete documentation index, see llms.txt. Markdown versions of all pages are available by appending .md to any URL (e.g. /get-started.md).
Mojo struct
Conv2dFpropKernel
struct Conv2dFpropKernel[act_type: DType, filter_type: DType, out_type: DType, config: Conv2dConfig[act_type, filter_type, out_type], cluster_shape: Coord[Int, Int, Int] = Coord(Int(1), Int(1), Int(1)), register_based_epilogue: Bool = True]
SM100 Conv2D forward propagation kernel.
This kernel implements conv2d fprop using implicit GEMM with warp specialization. It reuses the matmul kernel architecture but with convolution-specific address calculation.
The kernel structure:
- Scheduler warp: CLC-based tile scheduling
- Load warp: TMA loads with im2col transformation
- MMA warp: Tensor core operations
- Epilogue warps: Output from TMEM to GMEM
Parameters
- act_type (
DType): Activation data type. - filter_type (
DType): Filter data type. - out_type (
DType): Output data type. - config (
Conv2dConfig[act_type, filter_type, out_type]): Kernel configuration. - cluster_shape (
Coord[Int, Int, Int]): CUDA cluster dimensions. - register_based_epilogue (
Bool): Whether to apply the epilogue in registers.
Implemented traits
comptime members
accum_layout
comptime accum_layout = row_major[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_N]()
accum_pipeline_consumer_arv_count
comptime accum_pipeline_consumer_arv_count = (config * Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4)))
accum_pipeline_producer_arv_count
comptime accum_pipeline_producer_arv_count = 1
accum_type
comptime accum_type = Conv2dConfig.accum_type()
AccumTensor
comptime AccumTensor = TmemTensor[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].accum_type, Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_N]](), TypeList[ComptimeInt[(load_from_mem config.mma_shape.__getitem_param__[Int(1)]())], ComptimeInt[Int(1)]]()], Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].accum_layout, cta_group=config.cta_group]
act_expected_bytes
comptime act_expected_bytes = (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.act_smem_elements * size_of[act_type]())
act_swizzle_elems
comptime act_swizzle_elems = (config.a_swizzle.bytes() // size_of[act_type]())
act_tile_dim0
comptime act_tile_dim0 = (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BM // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))
act_tma_load_size
comptime act_tma_load_size = (((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())) * (config.a_swizzle.bytes() // size_of[act_type]()))
act_tma_rows
comptime act_tma_rows = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].act_tile_dim0
ActDescLayout
comptime ActDescLayout = static_row_major[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())), (config.a_swizzle.bytes() // size_of[act_type]())]
ActTileLayout
comptime ActTileLayout = static_row_major[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())), Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK]
ActTileLoaderTypeIm2col
comptime ActTileLoaderTypeIm2col = TileLoaderTMAIm2col[tma_origin, dtype, tile_shape, desc_shape, cta_group=config.cta_group]
ActTmaOp
comptime ActTmaOp = TmaOpTypeIm2col[act_type, Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK]](), TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK], ComptimeInt[Int(1)]]()], Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]
BK
comptime BK = (load_from_mem config.block_tile_shape.__getitem_param__[Int(2)]())
BM
comptime BM = (load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]())
BN
comptime BN = (load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]())
clc_consumer_arv_count
comptime clc_consumer_arv_count = (DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size + Int((mul (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()), (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()), DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 7)))
clc_producer_arv_count
comptime clc_producer_arv_count = 1
clc_throttle_consumer_arv_count
comptime clc_throttle_consumer_arv_count = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SCHEDULER_THREADS
clc_throttle_producer_arv_count
comptime clc_throttle_producer_arv_count = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].TMA_LOAD_THREADS
CLUSTER_M
comptime CLUSTER_M = (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]())
CLUSTER_N
comptime CLUSTER_N = (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())
CLUSTER_SIZE
comptime CLUSTER_SIZE = ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) * (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))
Context
comptime Context = KernelContext[config.num_clc_pipeline_stages, config.cta_group, (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()), (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())]
cta_group
comptime cta_group = config.cta_group
epi_load_consumer_arv_count
comptime epi_load_consumer_arv_count = SIMD(Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4)))
epi_load_producer_arv_count
comptime epi_load_producer_arv_count = Int32(1)
EpiLoadPipelineType
comptime EpiLoadPipelineType = EpiLoadPipeline[((load_from_mem config.mma_shape.__getitem_param__[Int(1)]()) // (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]()))]
EPILOGUE_LOAD_THREADS
comptime EPILOGUE_LOAD_THREADS = WARP_SIZE
EPILOGUE_THREADS
comptime EPILOGUE_THREADS = (Int(4) * DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size)
EpilogueCtx
comptime EpilogueCtx = EpilogueWarpContext[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].opc, DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4))]
filter_expected_bytes
comptime filter_expected_bytes = (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.filter_smem_elements * size_of[filter_type]())
filter_swizzle_elems
comptime filter_swizzle_elems = (config.b_swizzle.bytes() // size_of[filter_type]())
filter_tile_dim0
comptime filter_tile_dim0 = (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BN // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))
filter_tma_load_size
comptime filter_tma_load_size = (((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config)) * (config.b_swizzle.bytes() // size_of[filter_type]()))
filter_tma_rows
comptime filter_tma_rows = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].filter_tile_dim0
FilterDescLayout
comptime FilterDescLayout = static_row_major[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config)), (config.b_swizzle.bytes() // size_of[filter_type]())]
FilterTileLayout
comptime FilterTileLayout = static_row_major[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config)), Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK]
FilterTileLoaderType
comptime FilterTileLoaderType = TileLoaderTMA[tma_origin, dtype, tile_shape, desc_shape, cta_group=config.cta_group]
FilterTmaOp
comptime FilterTmaOp = TmaOpType[filter_type, Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK]](), TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK], ComptimeInt[Int(1)]]()], Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]
input_expected_bytes
comptime input_expected_bytes = (Int((add (mul (config.a_swizzle.bytes() // size_of[act_type]()), ((load_from_mem config.block_tile_shape.__getitem_param__[Int(2)]()) // (config.a_swizzle.bytes() // size_of[act_type]())), ((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // Int(8)), size_of[act_type](), config.cta_group, 8), (mul (config.b_swizzle.bytes() // size_of[filter_type]()), ((load_from_mem config.block_tile_shape.__getitem_param__[Int(2)]()) // (config.b_swizzle.bytes() // size_of[filter_type]())), ((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // Int(8)), size_of[filter_type](), config.cta_group, 8))) * config)
InputTilePipelineType
comptime InputTilePipelineType = InputTilePipeline[StandardTilePayload[act_type, filter_type, Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BN, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), config.num_pipeline_stages], (config // config), config.k_group_size]
MMA_K
comptime MMA_K = (load_from_mem config.mma_shape.__getitem_param__[Int(2)]())
MMA_M
comptime MMA_M = (load_from_mem config.mma_shape.__getitem_param__[Int(0)]())
MMA_N
comptime MMA_N = (load_from_mem config.mma_shape.__getitem_param__[Int(1)]())
MMA_THREADS
comptime MMA_THREADS = WARP_SIZE
MmaCtx
comptime MmaCtx = MmaWarpContext[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].opc, DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4))]
MmaEpilogueSync
comptime MmaEpilogueSync = WarpGroupBarrier[(DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size + Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4))), Int(1)]
MmaOp
comptime MmaOp = MmaOpSM100_SS[out_type, act_type, filter_type, config.block_tile_shape, config.mma_shape, accum_type=Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].accum_type, cta_group=config.cta_group, cluster_shape=config.cluster_shape, a_swizzle=config.a_swizzle, b_swizzle=config.b_swizzle, transpose_b=True]
num_accum_pipeline_stages
comptime num_accum_pipeline_stages = config.num_accum_pipeline_stages
num_clc_pipeline_stages
comptime num_clc_pipeline_stages = config.num_clc_pipeline_stages
num_epi_load_stages
comptime num_epi_load_stages = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.num_epi_load_stages
num_group_pipeline_stages
comptime num_group_pipeline_stages = (config // config)
num_output_stages
comptime num_output_stages = config.num_output_stages
num_output_warps
comptime num_output_warps = 4
num_pipeline_stages
comptime num_pipeline_stages = config.num_pipeline_stages
NUM_THREADS
comptime NUM_THREADS = (Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4)) + Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4)))
NUM_TMEM_COLS
comptime NUM_TMEM_COLS = 512
opc
comptime opc = OutputPipelineConfig(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].num_accum_pipeline_stages, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].stage_stride_cols, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].cta_group)
out_swizzle_elems
comptime out_swizzle_elems = (config.c_swizzle.bytes() // size_of[out_type]())
out_tile_dim0
comptime out_tile_dim0 = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputM if (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M == Int(256)) if (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M == Int(256)) else (config == Int(1)) else Int(64)
OutDescLayout
comptime OutDescLayout = static_row_major[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0, (config.c_swizzle.bytes() // size_of[out_type]())]
OutputM
comptime OutputM = (load_from_mem config.output_tile_shape.__getitem_param__[Int(0)]())
OutputN
comptime OutputN = (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]())
OutputPipeline
comptime OutputPipeline = OutputTilePipeline[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].opc]
OutTileLayout
comptime OutTileLayout = static_row_major[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN]
OutTmaOp
comptime OutTmaOp = TmaOpType[out_type, Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN]](), TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN], ComptimeInt[Int(1)]]()], Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]
Scheduler
comptime Scheduler = TileScheduler[config.num_clc_pipeline_stages, config.cluster_shape, block_swizzle_size=config.block_swizzle_size]
SCHEDULER_THREADS
comptime SCHEDULER_THREADS = WARP_SIZE
SmemType
comptime SmemType = Conv2dSmem[act_type, filter_type, out_type, config=config]
src_expected_bytes
comptime src_expected_bytes = ((Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputM * Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN) * size_of[out_type]())
SrcCTileArray
comptime SrcCTileArray = SMemTileArray2DRowMajor[out_type, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.OutputM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.OutputN, ((load_from_mem config.mma_shape.__getitem_param__[Int(1)]()) // (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]()))]
SrcDescLayout
comptime SrcDescLayout = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutDescLayout
SrcTileLayout
comptime SrcTileLayout = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutTileLayout
SrcTmaOp
comptime SrcTmaOp = TmaOpType[out_type, Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN]](), TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN], ComptimeInt[Int(1)]]()], Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]
stage_stride_cols
comptime stage_stride_cols = (Int(512) // config)
TilePayload
comptime TilePayload = StandardTilePayload[act_type, filter_type, Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BN, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), config.num_pipeline_stages]
TileWriterType
comptime TileWriterType = TileWriter[act_type, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].accum_type, Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BN, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_N, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_K), Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].opc, config.c_swizzle, False, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.OutputM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.OutputN, config.num_output_stages, Int(4), register_based_epilogue]
TMA_LOAD_THREADS
comptime TMA_LOAD_THREADS = WARP_SIZE
Tmem
comptime Tmem = TmemAllocation[OutputPipelineConfig(config.num_accum_pipeline_stages, (Int(512) // config), config.cta_group).cta_group]
TmemDealloc
comptime TmemDealloc = TmemDeallocBarrier[OutputPipelineConfig(config.num_accum_pipeline_stages, (Int(512) // config), config.cta_group).cta_group]
Methods
mma
static def mma[tiles_origin: MutOrigin, //](tmem_stage: TmemStage[Self.opc], tiles: ConsumerTiles[tiles_origin, StandardTilePayload[act_type, filter_type, Tuple(Self.BM, Self.BK), Tuple(Self.BN, Self.BK), config.num_pipeline_stages], (config // config), config.k_group_size], mma_op: MmaOpSM100_SS[accum_type=mma_op.accum_type, cta_group=mma_op.cta_group, cluster_shape=mma_op.cluster_shape, a_swizzle=mma_op.a_swizzle, b_swizzle=mma_op.b_swizzle, transpose_b=mma_op.transpose_b], elect_one_warp: Bool, iter_idx: UInt32, k_start: UInt32)
Execute MMA operations for one pipeline stage.
Parameters:
- tiles_origin (
MutOrigin): Borrow origin of the consumer tile payload (inferred).
Args:
- tmem_stage (
TmemStage[Self.opc]): TMEM accumulator stage to accumulate results into. - tiles (
ConsumerTiles[tiles_origin, StandardTilePayload[act_type, filter_type, Tuple(Self.BM, Self.BK), Tuple(Self.BN, Self.BK), config.num_pipeline_stages], (config // config), config.k_group_size]): Consumer tile payload and barrier handle holding the activation and filter tiles for the current stage. - mma_op (
MmaOpSM100_SS[accum_type=mma_op.accum_type, cta_group=mma_op.cta_group, cluster_shape=mma_op.cluster_shape, a_swizzle=mma_op.a_swizzle, b_swizzle=mma_op.b_swizzle, transpose_b=mma_op.transpose_b]): SM100 tensor core MMA operation object. - elect_one_warp (
Bool): Whether this warp is elected for cluster-wide operations. - iter_idx (
UInt32): K-dimension tile index of the current MMA iteration. - k_start (
UInt32): K-dimension tile index at which accumulation begins, used to initialize the accumulator on the first tile.
init_barriers
static def init_barriers(ctx: KernelContext[config.num_clc_pipeline_stages, config.cta_group, (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()), (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())], act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], input_barriers: SMemArray[SharedMemBarrier, ((config // config) * Int(2))], accum_barriers: SMemArray[SharedMemBarrier, (config * Int(2))], clc_throttle: SMemArray[SharedMemBarrier, (config * Int(2))], clc_full: SMemArray[SharedMemBarrier, config.num_clc_pipeline_stages], clc_empty: SMemArray[SharedMemBarrier, config.num_clc_pipeline_stages], tmem_dealloc: SMemArray[SharedMemBarrier, Int(1)], epi_load_barriers: SMemArray[SharedMemBarrier, (((load_from_mem config.mma_shape.__getitem_param__[Int(1)]()) // (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]())) * Int(2))], load_order_barrier: SMemArray[SharedMemBarrier, Int(1)])
Initialize barriers and prefetch TMA descriptors.
Args:
- ctx (
KernelContext[config.num_clc_pipeline_stages, config.cta_group, (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()), (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())]): Kernel context holding per-CTA state and multicast masks. - act_tma_op (
TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()]): Im2col TMA descriptor for the activation tensor. - filter_tma_op (
TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for the filter tensor. - out_tma_op (
TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for the output tensor. - input_barriers (
SMemArray[SharedMemBarrier, ((config // config) * Int(2))]): Producer-consumer barriers for the input tile pipeline. - accum_barriers (
SMemArray[SharedMemBarrier, (config * Int(2))]): Producer-consumer barriers for the TMEM accumulator pipeline. - clc_throttle (
SMemArray[SharedMemBarrier, (config * Int(2))]): Throttle barriers pacing the CLC scheduler against the load warps. - clc_full (
SMemArray[SharedMemBarrier, config.num_clc_pipeline_stages]): CLC barriers signalling tile slots are full, producer to consumer. - clc_empty (
SMemArray[SharedMemBarrier, config.num_clc_pipeline_stages]): CLC barriers signalling tile slots are empty, consumer to producer. - tmem_dealloc (
SMemArray[SharedMemBarrier, Int(1)]): Barrier governing TMEM accumulator deallocation. - epi_load_barriers (
SMemArray[SharedMemBarrier, (((load_from_mem config.mma_shape.__getitem_param__[Int(1)]()) // (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]())) * Int(2))]): Barriers for the epilogue load pipeline (source C residual loading). - load_order_barrier (
SMemArray[SharedMemBarrier, Int(1)]): Barrier coordinating MainLoad completion with EpilogueLoad.
load_input_tiles
static def load_input_tiles[act_tma_origin: ImmOrigin, filter_tma_origin: ImmOrigin, tiles_origin: MutOrigin, //](act_loader: TileLoaderTMAIm2col[act_tma_origin, act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]](), cta_group=config.cta_group], filter_loader: TileLoaderTMA[filter_tma_origin, filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]](), cta_group=config.cta_group], tiles: ProducerTiles[tiles_origin, StandardTilePayload[act_type, filter_type, Tuple(Self.BM, Self.BK), Tuple(Self.BN, Self.BK), config.num_pipeline_stages], (config // config), config.k_group_size], iter_idx: UInt32, work_m_coord: Int, work_n_coord: Int, peer_cta_coord: Tuple[Int, Int, Int], elect_one_cta: Bool)
Load activation (via im2col TMA) and filter tiles.
The im2col TMA descriptor handles coordinate transformation internally. Coordinates are in GEMM space.
Parameters:
- act_tma_origin (
ImmOrigin): Borrow origin of the activation TMA descriptor (inferred). - filter_tma_origin (
ImmOrigin): Borrow origin of the filter TMA descriptor (inferred). - tiles_origin (
MutOrigin): Borrow origin of the producer tile payload (inferred).
Args:
- act_loader (
TileLoaderTMAIm2col[act_tma_origin, act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]](), cta_group=config.cta_group]): Im2col TMA tile loader for activation tiles. - filter_loader (
TileLoaderTMA[filter_tma_origin, filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]](), cta_group=config.cta_group]): TMA tile loader for filter tiles. - tiles (
ProducerTiles[tiles_origin, StandardTilePayload[act_type, filter_type, Tuple(Self.BM, Self.BK), Tuple(Self.BN, Self.BK), config.num_pipeline_stages], (config // config), config.k_group_size]): Producer tile payload and barrier handle for the current stage. - iter_idx (
UInt32): K-dimension tile index (C * R * S) of the current load. - work_m_coord (
Int): M coordinate in GEMM space (batch * H_out * W_out). - work_n_coord (
Int): N coordinate in GEMM space (output channels). - peer_cta_coord (
Tuple[Int, Int, Int]): Peer CTA slicing coordinates as(peer_rank_n, peer_rank_m, peer_m_rank)for cluster-distributed tiles. - elect_one_cta (
Bool): Whether this CTA is elected for cluster-wide operations.
run
static def run(act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)])
Kernel entry point for Conv2D fprop (no residual).
Args:
- act_tma_op (
TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()]): Im2col TMA descriptor for activation. - filter_tma_op (
TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for filter. - out_tma_op (
TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for output. - cluster_dim (
StaticTuple[Int32, Int(3)]): Cluster dimensions. - mnk (
StaticTuple[UInt32, Int(3)]): GEMM dimensions (M, N, K).
run_with_compute_fn
static def run_with_compute_fn[ComputeFnType: ElementwiseComputeFn](act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)], compute_fn: ComputeFnType)
Kernel entry point for Conv2D fprop with a compute epilogue value.
Parameters:
- ComputeFnType (
ElementwiseComputeFn): Type of the compute epilogue closure.
Args:
- act_tma_op (
TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()]): Im2col TMA descriptor for activation. - filter_tma_op (
TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for filter. - out_tma_op (
TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for output. - cluster_dim (
StaticTuple[Int32, Int(3)]): Cluster dimensions. - mnk (
StaticTuple[UInt32, Int(3)]): GEMM dimensions (M, N, K). - compute_fn (
ComputeFnType): Element-wise epilogue applied to each output value.
run_with_epilogue_fn
static def run_with_epilogue_fn[EpilogueFnType: ElementwiseEpilogueFn](act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)], epilogue_fn: EpilogueFnType)
Kernel entry point for Conv2D fprop with an elementwise epilogue.
Parameters:
- EpilogueFnType (
ElementwiseEpilogueFn): Type of the elementwise epilogue closure.
Args:
- act_tma_op (
TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()]): Im2col TMA descriptor for activation. - filter_tma_op (
TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for filter. - out_tma_op (
TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for output. - cluster_dim (
StaticTuple[Int32, Int(3)]): Cluster dimensions. - mnk (
StaticTuple[UInt32, Int(3)]): GEMM dimensions (M, N, K). - epilogue_fn (
EpilogueFnType): Elementwise epilogue that stores each output value.
run_with_residual
static def run_with_residual(act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], src_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)], beta: Float32)
Kernel entry point for Conv2D fprop with residual (D = Conv + beta*C).
Args:
- act_tma_op (
TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()]): Im2col TMA descriptor for activation. - filter_tma_op (
TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for filter. - out_tma_op (
TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for output D. - src_tma_op (
TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for source C (residual input). - cluster_dim (
StaticTuple[Int32, Int(3)]): Cluster dimensions. - mnk (
StaticTuple[UInt32, Int(3)]): GEMM dimensions (M, N, K). - beta (
Float32): Residual scale factor.
run_with_residual_and_epilogue_fn
static def run_with_residual_and_epilogue_fn[EpilogueFnType: ElementwiseEpilogueFn](act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], src_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)], beta: Float32, epilogue_fn: EpilogueFnType)
Kernel entry point for Conv2D fprop with residual and an epilogue.
Parameters:
- EpilogueFnType (
ElementwiseEpilogueFn): Type of the elementwise epilogue closure.
Args:
- act_tma_op (
TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()]): Im2col TMA descriptor for activation. - filter_tma_op (
TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for filter. - out_tma_op (
TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for output D. - src_tma_op (
TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()]): TMA descriptor for source C (residual input). - cluster_dim (
StaticTuple[Int32, Int(3)]): Cluster dimensions. - mnk (
StaticTuple[UInt32, Int(3)]): GEMM dimensions (M, N, K). - beta (
Float32): Residual scale factor. - epilogue_fn (
EpilogueFnType): Elementwise epilogue that stores each output value after the residual add.