For the complete documentation index, see llms.txt. Markdown versions of all pages are available by appending .md to any URL (e.g. /get-started.md).
Mojo struct
SM100MLA
struct SM100MLA
SM100 (Blackwell datacenter) MLA kernels.
Implemented traitsβ
AnyType,
Deinitable,
MLAKernels,
Movable
Methodsβ
decode_dispatchβ
static def decode_dispatch[k_t: MHAOperand, mask_t: MHAMask, dtype: DType, //, kv_num_heads: Int, config: MHAConfig[dtype], ragged: Bool = False, _is_cache_length_accurate: Bool = False, _use_valid_length: Bool = True, decoding_warp_split_k: Bool = False, per_token_scale_rope_aware: Bool = False, sparse: Bool = False, fold_shared_index: Bool = False, has_extra_k: Bool = False, physical_indices: Bool = True, AttnSinkPtrType: OptionalPointer = NullPointer[.float32], TopkLengthsPtrType: OptionalPointer = NullPointer[.int32], sparse_row: SparseRowLayout = SparseRowLayout(), LseOutPtrType: OptionalPointer = NullPointer[.float32], *, skip_masked_tiles: Bool = False](output: TileTensor[Engine=output.Engine, linear_idx_type=output.linear_idx_type], q: TileTensor[dtype, Engine=q.Engine, linear_idx_type=q.linear_idx_type], k: k_t, mask_functor: mask_t, valid_length: TileTensor[.uint32, Engine=valid_length.Engine, linear_idx_type=valid_length.linear_idx_type], max_prompt_len: Int, max_cache_valid_length: Int, scale: Float32, ctx: DeviceContext, scalar_args_buf: NullableTileTensor[.int64, Engine=scalar_args_buf.Engine, linear_idx_type=scalar_args_buf.linear_idx_type], kv_input_row_offsets: OptionalReg[TileTensor[.uint32, Layout[TypeList[Int64](), TypeList[ComptimeInt[Int(1)]]()], ImmutAnyOrigin]] = None, num_partitions: Optional[Int] = None, q_scale_ptr: OptionalReg[Pointer[Float32, MutAnyOrigin]] = None, d_indices: OptionalReg[Pointer[Int32, MutAnyOrigin]] = None, indices_stride: Int = Int(0), topk_lengths: TopkLengthsPtrType = null_pointer[TopkLengthsPtrType](), attn_sink_ptr: AttnSinkPtrType = null_pointer[AttnSinkPtrType](), extra_k: OptionalReg[k_t] = None, extra_d_indices: OptionalReg[Pointer[Int32, MutAnyOrigin]] = None, extra_indices_stride: Int = Int(0), extra_topk_lengths: TopkLengthsPtrType = unread_pointer[TopkLengthsPtrType](), extra_scales_ptr: OptionalReg[Pointer[Float32, MutAnyOrigin]] = None, num_partitions_in: Optional[Int] = None, logical_indices: OptionalReg[Pointer[Int32, MutAnyOrigin]] = None, selection_lag: Int = Int(0), lse_out: LseOutPtrType = null_pointer[LseOutPtrType](), split_len_bound: Int = Int(0), row_invariant_verify: Bool = False)
Dispatches an MLA decoding request to the SM100 kernels.
prefill_dispatchβ
static def prefill_dispatch[k_t: MHAOperand, v_t: MHAOperand, k_rope_t: MHAOperand, mask_t: MHAMask, dtype: DType, output_type: DType, //, kv_num_heads: Int, config: MHAConfig[dtype], q_depth: Int = Int(192), cache_depth: Int = Int(576), _ndbuffer_mha_operand: Bool = False](output: TileTensor[output_type, Engine=output.Engine, linear_idx_type=output.linear_idx_type], q: TileTensor[dtype, Engine=q.Engine, linear_idx_type=q.linear_idx_type], k: k_t, v: v_t, k_rope: k_rope_t, mask_functor: mask_t, valid_length: TileTensor[.uint32, Engine=valid_length.Engine, linear_idx_type=valid_length.linear_idx_type], max_prompt_len: Int, scale: Float32, ctx: DeviceContext, cache_offsets: OptionalReg[TileTensor[.uint32, Layout[TypeList[Int64](), TypeList[ComptimeInt[Int(1)]]()], ImmutAnyOrigin]] = None)
Dispatches an MLA prefill request to the SM100 kernels.