IMPORTANT: To view this page as Markdown, append `.md` to the URL (e.g. /get-started.md). For the complete documentation index, see llms.txt.
Skip to main content
For the complete documentation index, see llms.txt. Markdown versions of all pages are available by appending .md to any URL (e.g. /get-started.md).

Mojo struct

Conv2dFpropKernel

struct Conv2dFpropKernel[act_type: DType, filter_type: DType, out_type: DType, config: Conv2dConfig[act_type, filter_type, out_type], cluster_shape: Coord[Int, Int, Int] = Coord(Int(1), Int(1), Int(1)), register_based_epilogue: Bool = True]

SM100 Conv2D forward propagation kernel.

This kernel implements conv2d fprop using implicit GEMM with warp specialization. It reuses the matmul kernel architecture but with convolution-specific address calculation.

The kernel structure:

  • Scheduler warp: CLC-based tile scheduling
  • Load warp: TMA loads with im2col transformation
  • MMA warp: Tensor core operations
  • Epilogue warps: Output from TMEM to GMEM

Parameters​

Implemented traits​

AnyType, Deinitable, Movable

comptime members​

accum_layout​

comptime accum_layout = row_major[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_N]()

accum_pipeline_consumer_arv_count​

comptime accum_pipeline_consumer_arv_count = (config * Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4)))

accum_pipeline_producer_arv_count​

comptime accum_pipeline_producer_arv_count = 1

accum_type​

comptime accum_type = Conv2dConfig.accum_type()

AccumTensor​

comptime AccumTensor = TmemTensor[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].accum_type, Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_N]](), TypeList[ComptimeInt[(load_from_mem config.mma_shape.__getitem_param__[Int(1)]())], ComptimeInt[Int(1)]]()], Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].accum_layout, cta_group=config.cta_group]

act_expected_bytes​

comptime act_expected_bytes = (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.act_smem_elements * size_of[act_type]())

act_swizzle_elems​

comptime act_swizzle_elems = (config.a_swizzle.bytes() // size_of[act_type]())

act_tile_dim0​

comptime act_tile_dim0 = (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BM // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))

act_tma_load_size​

comptime act_tma_load_size = (((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())) * (config.a_swizzle.bytes() // size_of[act_type]()))

act_tma_rows​

comptime act_tma_rows = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].act_tile_dim0

ActDescLayout​

comptime ActDescLayout = static_row_major[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())), (config.a_swizzle.bytes() // size_of[act_type]())]

ActTileLayout​

comptime ActTileLayout = static_row_major[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())), Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK]

ActTileLoaderTypeIm2col​

comptime ActTileLoaderTypeIm2col = TileLoaderTMAIm2col[tma_origin, dtype, tile_shape, desc_shape, cta_group=config.cta_group]

ActTmaOp​

comptime ActTmaOp = TmaOpTypeIm2col[act_type, Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK]](), TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK], ComptimeInt[Int(1)]]()], Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]

BK​

comptime BK = (load_from_mem config.block_tile_shape.__getitem_param__[Int(2)]())

BM​

comptime BM = (load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]())

BN​

comptime BN = (load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]())

clc_consumer_arv_count​

comptime clc_consumer_arv_count = (DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size + Int((mul (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()), (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()), DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 7)))

clc_producer_arv_count​

comptime clc_producer_arv_count = 1

clc_throttle_consumer_arv_count​

comptime clc_throttle_consumer_arv_count = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SCHEDULER_THREADS

clc_throttle_producer_arv_count​

comptime clc_throttle_producer_arv_count = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].TMA_LOAD_THREADS

CLUSTER_M​

comptime CLUSTER_M = (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]())

CLUSTER_N​

comptime CLUSTER_N = (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())

CLUSTER_SIZE​

comptime CLUSTER_SIZE = ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) * (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))

Context​

comptime Context = KernelContext[config.num_clc_pipeline_stages, config.cta_group, (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()), (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())]

cta_group​

comptime cta_group = config.cta_group

epi_load_consumer_arv_count​

comptime epi_load_consumer_arv_count = SIMD(Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4)))

epi_load_producer_arv_count​

comptime epi_load_producer_arv_count = Int32(1)

EpiLoadPipelineType​

comptime EpiLoadPipelineType = EpiLoadPipeline[((load_from_mem config.mma_shape.__getitem_param__[Int(1)]()) // (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]()))]

EPILOGUE_LOAD_THREADS​

comptime EPILOGUE_LOAD_THREADS = WARP_SIZE

EPILOGUE_THREADS​

comptime EPILOGUE_THREADS = (Int(4) * DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size)

EpilogueCtx​

comptime EpilogueCtx = EpilogueWarpContext[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].opc, DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4))]

filter_expected_bytes​

comptime filter_expected_bytes = (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.filter_smem_elements * size_of[filter_type]())

filter_swizzle_elems​

comptime filter_swizzle_elems = (config.b_swizzle.bytes() // size_of[filter_type]())

filter_tile_dim0​

comptime filter_tile_dim0 = (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BN // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))

filter_tma_load_size​

comptime filter_tma_load_size = (((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config)) * (config.b_swizzle.bytes() // size_of[filter_type]()))

filter_tma_rows​

comptime filter_tma_rows = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].filter_tile_dim0

FilterDescLayout​

comptime FilterDescLayout = static_row_major[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config)), (config.b_swizzle.bytes() // size_of[filter_type]())]

FilterTileLayout​

comptime FilterTileLayout = static_row_major[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config)), Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK]

FilterTileLoaderType​

comptime FilterTileLoaderType = TileLoaderTMA[tma_origin, dtype, tile_shape, desc_shape, cta_group=config.cta_group]

FilterTmaOp​

comptime FilterTmaOp = TmaOpType[filter_type, Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK]](), TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK], ComptimeInt[Int(1)]]()], Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]

input_expected_bytes​

comptime input_expected_bytes = (Int((add (mul (config.a_swizzle.bytes() // size_of[act_type]()), ((load_from_mem config.block_tile_shape.__getitem_param__[Int(2)]()) // (config.a_swizzle.bytes() // size_of[act_type]())), ((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // Int(8)), size_of[act_type](), config.cta_group, 8), (mul (config.b_swizzle.bytes() // size_of[filter_type]()), ((load_from_mem config.block_tile_shape.__getitem_param__[Int(2)]()) // (config.b_swizzle.bytes() // size_of[filter_type]())), ((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // Int(8)), size_of[filter_type](), config.cta_group, 8))) * config)

InputTilePipelineType​

comptime InputTilePipelineType = InputTilePipeline[StandardTilePayload[act_type, filter_type, Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BN, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), config.num_pipeline_stages], (config // config), config.k_group_size]

MMA_K​

comptime MMA_K = (load_from_mem config.mma_shape.__getitem_param__[Int(2)]())

MMA_M​

comptime MMA_M = (load_from_mem config.mma_shape.__getitem_param__[Int(0)]())

MMA_N​

comptime MMA_N = (load_from_mem config.mma_shape.__getitem_param__[Int(1)]())

MMA_THREADS​

comptime MMA_THREADS = WARP_SIZE

MmaCtx​

comptime MmaCtx = MmaWarpContext[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].opc, DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4))]

MmaEpilogueSync​

comptime MmaEpilogueSync = WarpGroupBarrier[(DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size + Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4))), Int(1)]

MmaOp​

comptime MmaOp = MmaOpSM100_SS[out_type, act_type, filter_type, config.block_tile_shape, config.mma_shape, accum_type=Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].accum_type, cta_group=config.cta_group, cluster_shape=config.cluster_shape, a_swizzle=config.a_swizzle, b_swizzle=config.b_swizzle, transpose_b=True]

num_accum_pipeline_stages​

comptime num_accum_pipeline_stages = config.num_accum_pipeline_stages

num_clc_pipeline_stages​

comptime num_clc_pipeline_stages = config.num_clc_pipeline_stages

num_epi_load_stages​

comptime num_epi_load_stages = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.num_epi_load_stages

num_group_pipeline_stages​

comptime num_group_pipeline_stages = (config // config)

num_output_stages​

comptime num_output_stages = config.num_output_stages

num_output_warps​

comptime num_output_warps = 4

num_pipeline_stages​

comptime num_pipeline_stages = config.num_pipeline_stages

NUM_THREADS​

comptime NUM_THREADS = (Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4)) + Int((mul DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size, 4)))

NUM_TMEM_COLS​

comptime NUM_TMEM_COLS = 512

opc​

comptime opc = OutputPipelineConfig(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].num_accum_pipeline_stages, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].stage_stride_cols, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].cta_group)

out_swizzle_elems​

comptime out_swizzle_elems = (config.c_swizzle.bytes() // size_of[out_type]())

out_tile_dim0​

comptime out_tile_dim0 = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputM if (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M == Int(256)) if (Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M == Int(256)) else (config == Int(1)) else Int(64)

OutDescLayout​

comptime OutDescLayout = static_row_major[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0, (config.c_swizzle.bytes() // size_of[out_type]())]

OutputM​

comptime OutputM = (load_from_mem config.output_tile_shape.__getitem_param__[Int(0)]())

OutputN​

comptime OutputN = (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]())

OutputPipeline​

comptime OutputPipeline = OutputTilePipeline[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].opc]

OutTileLayout​

comptime OutTileLayout = static_row_major[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN]

OutTmaOp​

comptime OutTmaOp = TmaOpType[out_type, Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN]](), TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN], ComptimeInt[Int(1)]]()], Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]

Scheduler​

comptime Scheduler = TileScheduler[config.num_clc_pipeline_stages, config.cluster_shape, block_swizzle_size=config.block_swizzle_size]

SCHEDULER_THREADS​

comptime SCHEDULER_THREADS = WARP_SIZE

SmemType​

comptime SmemType = Conv2dSmem[act_type, filter_type, out_type, config=config]

src_expected_bytes​

comptime src_expected_bytes = ((Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputM * Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN) * size_of[out_type]())

SrcCTileArray​

comptime SrcCTileArray = SMemTileArray2DRowMajor[out_type, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.OutputM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.OutputN, ((load_from_mem config.mma_shape.__getitem_param__[Int(1)]()) // (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]()))]

SrcDescLayout​

comptime SrcDescLayout = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutDescLayout

SrcTileLayout​

comptime SrcTileLayout = Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutTileLayout

SrcTmaOp​

comptime SrcTmaOp = TmaOpType[out_type, Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0], ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN]](), TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].OutputN], ComptimeInt[Int(1)]]()], Layout[TypeList[ComptimeInt[Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]

stage_stride_cols​

comptime stage_stride_cols = (Int(512) // config)

TilePayload​

comptime TilePayload = StandardTilePayload[act_type, filter_type, Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BN, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), config.num_pipeline_stages]

TileWriterType​

comptime TileWriterType = TileWriter[act_type, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].accum_type, Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BN, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].BK), Tuple(Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_M, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_N, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].MMA_K), Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].opc, config.c_swizzle, False, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.OutputM, Conv2dFpropKernel[act_type, filter_type, out_type, config, cluster_shape, register_based_epilogue].SmemType.OutputN, config.num_output_stages, Int(4), register_based_epilogue]

TMA_LOAD_THREADS​

comptime TMA_LOAD_THREADS = WARP_SIZE

Tmem​

comptime Tmem = TmemAllocation[OutputPipelineConfig(config.num_accum_pipeline_stages, (Int(512) // config), config.cta_group).cta_group]

TmemDealloc​

comptime TmemDealloc = TmemDeallocBarrier[OutputPipelineConfig(config.num_accum_pipeline_stages, (Int(512) // config), config.cta_group).cta_group]

Methods​

mma​

static def mma[tiles_origin: MutOrigin, //](tmem_stage: TmemStage[Self.opc], tiles: ConsumerTiles[tiles_origin, StandardTilePayload[act_type, filter_type, Tuple(Self.BM, Self.BK), Tuple(Self.BN, Self.BK), config.num_pipeline_stages], (config // config), config.k_group_size], mma_op: MmaOpSM100_SS[accum_type=mma_op.accum_type, cta_group=mma_op.cta_group, cluster_shape=mma_op.cluster_shape, a_swizzle=mma_op.a_swizzle, b_swizzle=mma_op.b_swizzle, transpose_b=mma_op.transpose_b], elect_one_warp: Bool, iter_idx: UInt32, k_start: UInt32)

Execute MMA operations for one pipeline stage.

Parameters:

  • ​tiles_origin (MutOrigin): Borrow origin of the consumer tile payload (inferred).

Args:

init_barriers​

static def init_barriers(ctx: KernelContext[config.num_clc_pipeline_stages, config.cta_group, (load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()), (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]())], act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], input_barriers: SMemArray[SharedMemBarrier, ((config // config) * Int(2))], accum_barriers: SMemArray[SharedMemBarrier, (config * Int(2))], clc_throttle: SMemArray[SharedMemBarrier, (config * Int(2))], clc_full: SMemArray[SharedMemBarrier, config.num_clc_pipeline_stages], clc_empty: SMemArray[SharedMemBarrier, config.num_clc_pipeline_stages], tmem_dealloc: SMemArray[SharedMemBarrier, Int(1)], epi_load_barriers: SMemArray[SharedMemBarrier, (((load_from_mem config.mma_shape.__getitem_param__[Int(1)]()) // (load_from_mem config.output_tile_shape.__getitem_param__[Int(1)]())) * Int(2))], load_order_barrier: SMemArray[SharedMemBarrier, Int(1)])

Initialize barriers and prefetch TMA descriptors.

Args:

load_input_tiles​

static def load_input_tiles[act_tma_origin: ImmOrigin, filter_tma_origin: ImmOrigin, tiles_origin: MutOrigin, //](act_loader: TileLoaderTMAIm2col[act_tma_origin, act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]](), cta_group=config.cta_group], filter_loader: TileLoaderTMA[filter_tma_origin, filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]](), cta_group=config.cta_group], tiles: ProducerTiles[tiles_origin, StandardTilePayload[act_type, filter_type, Tuple(Self.BM, Self.BK), Tuple(Self.BN, Self.BK), config.num_pipeline_stages], (config // config), config.k_group_size], iter_idx: UInt32, work_m_coord: Int, work_n_coord: Int, peer_cta_coord: Tuple[Int, Int, Int], elect_one_cta: Bool)

Load activation (via im2col TMA) and filter tiles.

The im2col TMA descriptor handles coordinate transformation internally. Coordinates are in GEMM space.

Parameters:

  • ​act_tma_origin (ImmOrigin): Borrow origin of the activation TMA descriptor (inferred).
  • ​filter_tma_origin (ImmOrigin): Borrow origin of the filter TMA descriptor (inferred).
  • ​tiles_origin (MutOrigin): Borrow origin of the producer tile payload (inferred).

Args:

run​

static def run(act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)])

Kernel entry point for Conv2D fprop (no residual).

Args:

run_with_compute_fn​

static def run_with_compute_fn[ComputeFnType: ElementwiseComputeFn](act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)], compute_fn: ComputeFnType)

Kernel entry point for Conv2D fprop with a compute epilogue value.

Parameters:

Args:

run_with_epilogue_fn​

static def run_with_epilogue_fn[EpilogueFnType: ElementwiseEpilogueFn](act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)], epilogue_fn: EpilogueFnType)

Kernel entry point for Conv2D fprop with an elementwise epilogue.

Parameters:

Args:

run_with_residual​

static def run_with_residual(act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], src_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)], beta: Float32)

Kernel entry point for Conv2D fprop with residual (D = Conv + beta*C).

Args:

run_with_residual_and_epilogue_fn​

static def run_with_residual_and_epilogue_fn[EpilogueFnType: ElementwiseEpilogueFn](act_tma_op: TMATensorTileIm2col[act_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(0)]()) // (load_from_mem config.cluster_shape.__getitem_param__[Int(1)]()))], ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())]](), TypeList[ComptimeInt[(config.a_swizzle.bytes() // size_of[act_type]())], ComptimeInt[Int(1)]]()]]()], filter_tma_op: TMATensorTile[filter_type, _to_coord[Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[Self.BK]](), TypeList[ComptimeInt[Self.BK], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[((load_from_mem config.block_tile_shape.__getitem_param__[Int(1)]()) // ((load_from_mem config.cluster_shape.__getitem_param__[Int(0)]()) // config))], ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())]](), TypeList[ComptimeInt[(config.b_swizzle.bytes() // size_of[filter_type]())], ComptimeInt[Int(1)]]()]]()], out_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], src_tma_op: TMATensorTile[out_type, _to_coord[Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[Self.OutputN]](), TypeList[ComptimeInt[Self.OutputN], ComptimeInt[Int(1)]]()]](), _to_coord[Int(2), Layout[TypeList[ComptimeInt[Self.out_tile_dim0], ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())]](), TypeList[ComptimeInt[(config.c_swizzle.bytes() // size_of[out_type]())], ComptimeInt[Int(1)]]()]]()], cluster_dim: StaticTuple[Int32, Int(3)], mnk: StaticTuple[UInt32, Int(3)], beta: Float32, epilogue_fn: EpilogueFnType)

Kernel entry point for Conv2D fprop with residual and an epilogue.

Parameters:

Args:

Was this page helpful?