For the complete documentation index, see llms.txt. Markdown versions of all pages are available by appending .md to any URL (e.g. /max/get-started.md).

Mojo function

generic_get_paged_cache

def generic_get_paged_cache[dtype: DType](blocks: ManagedTensorSlice[IOSpec[_, _].MutableInput, static_spec=blocks.static_spec], cache_lengths: ManagedTensorSlice[IOSpec[_, _].Input, static_spec=cache_lengths.static_spec], lookup_table: ManagedTensorSlice[IOSpec[_, _].Input, static_spec=lookup_table.static_spec], max_prompt_length: ManagedTensorSlice[IOSpec[_, _].Input, static_spec=max_prompt_length.static_spec], max_cache_length: ManagedTensorSlice[IOSpec[_, _].Input, static_spec=max_cache_length.static_spec], out result: PagedKVCacheCollection[dtype, KVCacheStaticParams(SIMD[IntTuple](StaticTensorSpec[dtype, Int(6), blocks.static_spec.static_layout, blocks.static_spec.InFusion, blocks.static_spec.OutFusion, blocks.static_spec.ComputeFusion, blocks.static_spec.ComputeFusionTile, blocks.static_spec.OutFusionTile].shape_tuple[4]), SIMD[IntTuple](StaticTensorSpec[dtype, Int(6), blocks.static_spec.static_layout, blocks.static_spec.InFusion, blocks.static_spec.OutFusion, blocks.static_spec.ComputeFusion, blocks.static_spec.ComputeFusionTile, blocks.static_spec.OutFusionTile].shape_tuple[5]), (SIMD[IntTuple](StaticTensorSpec[dtype, Int(6), blocks.static_spec.static_layout, blocks.static_spec.InFusion, blocks.static_spec.OutFusion, blocks.static_spec.ComputeFusion, blocks.static_spec.ComputeFusionTile, blocks.static_spec.OutFusionTile].shape_tuple[1]) == Int(1))), SIMD[IntTuple](StaticTensorSpec[dtype, Int(6), blocks.static_spec.static_layout, blocks.static_spec.InFusion, blocks.static_spec.OutFusion, blocks.static_spec.ComputeFusion, blocks.static_spec.ComputeFusionTile, blocks.static_spec.OutFusionTile].shape_tuple[3]), MutAnyOrigin, ImmutAnyOrigin, ImmutAnyOrigin, MutUntrackedOrigin])

Returns:

def generic_get_paged_cache[dtype: DType, kv_params: KVCacheStaticParams, page_size: Int](blocks: LayoutTensor[dtype, Layout.row_major[Int(6)]()], cache_lengths: LayoutTensor[DType.uint32, Layout(IntTuple(Int(-1)))], lookup_table: LayoutTensor[DType.uint32, Layout.row_major[Int(2)]()], max_prompt_length: LayoutTensor[DType.uint32, Layout.row_major[Int(1)]()], max_cache_length: LayoutTensor[DType.uint32, Layout.row_major[Int(1)]()], out result: PagedKVCacheCollection[dtype, kv_params, page_size, blocks.origin, cache_lengths.origin, lookup_table.origin, MutUntrackedOrigin])

Returns:

PagedKVCacheCollection[dtype, kv_params, page_size, blocks.origin, cache_lengths.origin, lookup_table.origin, MutUntrackedOrigin]