For the complete documentation index, see llms.txt. Markdown versions of all pages are available by appending .md to any URL (e.g. /get-started.md).
Mojo struct
DecodeStreamingKVBuffer
struct DecodeStreamingKVBuffer[kv_t: MHAOperand, //, mma_shape: Tuple[Int, Int, Int], swizzle: Optional[Swizzle], BN: Int, WN: Int, BK: Int, num_threads: Int, depth: Int, kv_num_heads: Int, transpose: Bool]
Streaming-decode KV buffer: single-buffer SMEM staging with per-strip DMA.
Unlike KVBuffer, this takes an external DRAM tile per outer-loop iteration and loads BK-wide strips one at a time.
K (transpose=True): BN x BK SMEM, column strips from BN x depth. V (transpose=False): BK x depth SMEM (blocked BK x BK), row strips.
Parameters
- kv_t (
MHAOperand): MHA operand wrapping the KV cache pointer and dtype (inferred). - mma_shape (
Tuple[Int, Int, Int]): MFMA instruction shape[M, N, K]used for register tiling. - swizzle (
Optional[Swizzle]): LDS bank-conflict swizzle for SMEM loads and stores, orNone. - BN (
Int): Number of keys per DRAM tile (block tile extent along the key axis). - WN (
Int): Number of keys per warp tile (warp tile extent along the key axis). - BK (
Int): Strip width along the MMA K dimension of one per-iteration DRAM-to-SMEM load. - num_threads (
Int): Number of threads in the block driving DMA and register distribution. - depth (
Int): Per-head depth of the KV cache (head dimension). - kv_num_heads (
Int): Number of KV heads in the cache. - transpose (
Bool):Truefor the K operand (column strips fromBN x depth),Falsefor V (row strips).
Fields
- kv_mma_op (
DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].KVMmaOpType): - smem_ptr (
MutPointer[Scalar[kv_t.dtype], MutAnyOrigin, address_space=AddressSpace.SHARED]): - warp_id (
UInt32):
Implemented traits
comptime members
input_frag_size
comptime input_frag_size = ((DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].MMA_K * DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].MMA_N) // DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size)
KVMmaOpType
comptime KVMmaOpType = KVMmaOp[kv_t.dtype, mma_shape, DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].num_mmas, DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].num_k_mmas2, Int(1), BN, BK, transpose, swizzle]
MMA_K
comptime MMA_K = (load_from_mem mma_shape.__getitem_param__[Int(2)]())
mma_layout
comptime mma_layout = row_major[(DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].num_mmas * DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].num_k_mmas2), (Int((mul (load_from_mem mma_shape.__getitem_param__[Int(2)]()), (load_from_mem mma_shape.__getitem_param__[Int(1)]()))) // DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size)]()
MMA_N
comptime MMA_N = (load_from_mem mma_shape.__getitem_param__[Int(1)]())
MMATileType
comptime MMATileType = TileTensor[kv_t.dtype, Layout[TypeList[ComptimeInt[(DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].num_mmas * DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].num_k_mmas2)], ComptimeInt[(Int((mul (load_from_mem mma_shape.__getitem_param__[Int(2)]()), (load_from_mem mma_shape.__getitem_param__[Int(1)]()))) // DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size)]](), TypeList[ComptimeInt[(Int((mul (load_from_mem mma_shape.__getitem_param__[Int(2)]()), (load_from_mem mma_shape.__getitem_param__[Int(1)]()))) // DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> @std::@sys::@info::@"_current_target()"), :param_list<trait<@std::@traits::@anytype::@AnyType>> [@max::@_plugin::@_impl::@_NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size)], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]
num_k_mmas2
comptime num_k_mmas2 = ceildiv(BK, DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].MMA_K)
num_mmas
comptime num_mmas = ceildiv(WN if transpose else (depth // (BN // WN)), DecodeStreamingKVBuffer[mma_shape, swizzle, BN, WN, BK, num_threads, depth, kv_num_heads, transpose].MMA_N)
simd_width
comptime simd_width = simd_width_of[kv_t.dtype]()
warp_tile_rows
comptime warp_tile_rows = 32
Methods
__init__
def __init__(out self, cache: kv_t, batch_idx: Int, head_idx: Int, smem_ptr: MutPointer[Scalar[kv_t.dtype], MutAnyOrigin, address_space=AddressSpace.SHARED], num_keys: Int, warp_id: UInt32)
k_smem_block_tile
def k_smem_block_tile[tile_rows: Int](self, tile_row: Int) -> TileTensor[kv_t.dtype, Layout[TypeList[ComptimeInt[tile_rows], ComptimeInt[BK]](), TypeList[ComptimeInt[BK], ComptimeInt[Int(1)]]()], MutAnyOrigin, address_space=AddressSpace.SHARED]
Get a (tile_rows, BK) row-major sub-tile from the K SMEM.
Single-stage K SMEM has one BN×BK block, so only the row index along BN varies.
Parameters:
- tile_rows (
Int): Number of rows in the sub-tile.
Args:
- tile_row (
Int): Row index alongBNwithin theBN x BKSMEM block.
Returns:
load_from_dram
def load_from_dram[strip_idx: Int](self, gmem_tile: TileTensor[kv_t.dtype, Engine=gmem_tile.Engine, address_space=gmem_tile.address_space, linear_idx_type=gmem_tile.linear_idx_type])
Load one BK-wide strip from an external DRAM tile to SMEM.
K (transpose=True): columns [strip*BK, (strip+1)BK] from BN x depth. V (transpose=False): rows [stripBK, (strip+1)*BK] from BN x depth.
Parameters:
- strip_idx (
Int): Strip index along the K dimension (transpose) or the row axis (non-transpose) of the DRAM tile.
Args:
- gmem_tile (
TileTensor[kv_t.dtype, Engine=gmem_tile.Engine, address_space=gmem_tile.address_space, linear_idx_type=gmem_tile.linear_idx_type]): External DRAM tile to load the strip from.
load_from_shared
def load_from_shared(self)
Load from SMEM to MMA registers.
get_mma_tile
def get_mma_tile[k_mma_idx: Int](self) -> TileTensor[kv_t.dtype, Layout[TypeList[ComptimeInt[Self.num_mmas], ComptimeInt[(Int((mul (load_from_mem mma_shape.__getitem_param__[Int(2)]()), (load_from_mem mma_shape.__getitem_param__[Int(1)]()))) // DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> _std::_sys::_info::_"_current_target()"), :param_list<trait<_std::_traits::_anytype::_AnyType>> [_max::__plugin::__impl::__NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> _std::_sys::_info::_"_current_target()"), :param_list<trait<_std::_traits::_anytype::_AnyType>> [_max::__plugin::__impl::__NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> _std::_sys::_info::_"_current_target()"), :param_list<trait<_std::_traits::_anytype::_AnyType>> [_max::__plugin::__impl::__NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> _std::_sys::_info::_"_current_target()"), :param_list<trait<_std::_traits::_anytype::_AnyType>> [_max::__plugin::__impl::__NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size)]](), TypeList[ComptimeInt[(Int((mul (load_from_mem mma_shape.__getitem_param__[Int(2)]()), (load_from_mem mma_shape.__getitem_param__[Int(1)]()))) // DefaultMaxPlugin if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 0](), True) else [PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if (xor _index_lt[PluginSelector._find[_current_target(), 0, trap_if_missing=False](), 3](), True) else CudaMaxPlugin, HipMaxPlugin, MetalMaxPlugin[PluginSelector._find[_current_target(), 0, trap_if_missing=False]()] if identical(_NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> _std::_sys::_info::_"_current_target()"), :param_list<trait<_std::_traits::_anytype::_AnyType>> [_max::__plugin::__impl::__NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> _std::_sys::_info::_"_current_target()"), :param_list<trait<_std::_traits::_anytype::_AnyType>> [_max::__plugin::__impl::__NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin), _NoMaxPlugin) else _NoMaxPlugin if (xor _index_lt[TargetPluginSelector._find[0](), len(#kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> _std::_sys::_info::_"_current_target()"), :param_list<trait<_std::_traits::_anytype::_AnyType>> [_max::__plugin::__impl::__NoMaxPlugin]>)](), True) else #kgen.get_target_plugin<apply(:!lit.generator<() -> !kgen.target> _std::_sys::_info::_"_current_target()"), :param_list<trait<_std::_traits::_anytype::_AnyType>> [_max::__plugin::__impl::__NoMaxPlugin]>[TargetPluginSelector._find[0]()](MaxPlugin).HookFor[CompilationTarget(Int(0))].warp_size)], ComptimeInt[Int(1)]]()], MutUntrackedOrigin, address_space=AddressSpace.LOCAL]
Get register tile for one k_mma group within the single strip.
Parameters:
- k_mma_idx (
Int): Index of the MFMA tile along the K dimension within the strip.
Returns: