PTWM
ptwm

ptwm.preprocessing

Dtype-aware preprocessing for lossless weight compression.

ptwm.preprocessing

Dtype-aware preprocessing for lossless weight compression.

pymodule

ptwm.preprocessing

python/ptwm/preprocessing/__init__.py:1
module ptwm.preprocessing

Dtype-aware preprocessing for lossless weight compression.

The compression pipeline separates preprocessing (bit reorder + byte/nibble split) from entropy coding. Preprocessing is expressed as a typed DAG of ops (a :class:`Chain`) — see :mod:`._chains` for the chain builders and :mod:`._explorer` for the generative search that extends them.
pyfunction

ptwm.preprocessing.cache_dir

python/ptwm/preprocessing/_cache.py:67
def cache_dir() -> Path

Return the user-local chain cache directory (XDG-aware).

pyfunction

ptwm.preprocessing.cache_info

python/ptwm/preprocessing/_cache.py:176
def cache_info() -> dict[str, int]

Map cache filename → chain count, for diagnostics.

pyfunction

ptwm.preprocessing.cache_path_for

python/ptwm/preprocessing/_cache.py:74
def cache_path_for(dtype_code: int, role: ClassifierRole) -> Path
pyclass

ptwm.preprocessing.CacheEntry

python/ptwm/preprocessing/_cache.py:44
class CacheEntry:

One cached chain template, identified structurally.

pymethod

ptwm.preprocessing.CacheEntry.__init__

python/ptwm/preprocessing/_cache.py:1
def __init__(self, internal_dtype: int, ops: tuple[int, ...], byte_split_planes: int) -> None
pyattribute

ptwm.preprocessing.CacheEntry.byte_split_planes

python/ptwm/preprocessing/_cache.py:50
byte_split_planes: int
pyattribute

ptwm.preprocessing.CacheEntry.internal_dtype

python/ptwm/preprocessing/_cache.py:48
internal_dtype: int
pyattribute

ptwm.preprocessing.CacheEntry.ops

python/ptwm/preprocessing/_cache.py:49
ops: tuple[int, ...]
pymethod

ptwm.preprocessing.CacheEntry.to_builder

python/ptwm/preprocessing/_cache.py:52
def to_builder(self) -> ChainBuilder
pyclass

ptwm.preprocessing.Chain

python/ptwm/preprocessing/_chains.py:170
class Chain:
pyconstant

ptwm.preprocessing.CHAIN_BF16_SPLIT

python/ptwm/preprocessing/_chains.py:851
CHAIN_BF16_SPLIT: ChainBuilder = _bf16_split_chain

BF16 / FP16 standard: BitReorderIeee16 + ByteSplit(n=2).

pyconstant

ptwm.preprocessing.CHAIN_BYTE_PASSTHROUGH_VALUE

python/ptwm/preprocessing/_chains.py:925
CHAIN_BYTE_PASSTHROUGH_VALUE: ChainBuilder = _u8_byte_passthrough_value_chain

Uint8 standard passthrough.

pyconstant

ptwm.preprocessing.CHAIN_F32_GLOBAL_SCALE

python/ptwm/preprocessing/_chains.py:919
CHAIN_F32_GLOBAL_SCALE: ChainBuilder = _fp32_global_scale_chain

FP32 global-scale passthrough (NVFP4 input_scale tensors).

pyconstant

ptwm.preprocessing.CHAIN_F8E4M3_NIBBLE

python/ptwm/preprocessing/_chains.py:914
CHAIN_F8E4M3_NIBBLE: ChainBuilder = _fp8e4m3_nibble_chain
pyconstant

ptwm.preprocessing.CHAIN_F8E4M3_SCALE_PASSTHROUGH

python/ptwm/preprocessing/_chains.py:910
CHAIN_F8E4M3_SCALE_PASSTHROUGH: ChainBuilder = _fp8e4m3_scale_passthrough_chain

FP8 E4M3FN scale-block passthrough (enables O1SAC on block-scale tensors).

pyconstant

ptwm.preprocessing.CHAIN_FP32_SPLIT

python/ptwm/preprocessing/_chains.py:863
CHAIN_FP32_SPLIT: ChainBuilder = _fp32_split_chain

FP32 standard: BitReorderIeee32 + ByteSplit(n=4).

pyconstant

ptwm.preprocessing.CHAIN_FP4_NIBBLE_PASSTHROUGH

python/ptwm/preprocessing/_chains.py:922
CHAIN_FP4_NIBBLE_PASSTHROUGH: ChainBuilder = _fp4_passthrough_chain

Packed FP4 values passthrough (enables PerGroupCodebook).

pyconstant

ptwm.preprocessing.CHAIN_FP8E4M3_NIBBLE

python/ptwm/preprocessing/_chains.py:904
CHAIN_FP8E4M3_NIBBLE: ChainBuilder = _fp8e4m3_nibble_chain

FP8 E4M3FN standard: BitReorderFp8E4M3 + NibbleSplit.

pyconstant

ptwm.preprocessing.CHAIN_FP8E5M2_NIBBLE

python/ptwm/preprocessing/_chains.py:907
CHAIN_FP8E5M2_NIBBLE: ChainBuilder = _fp8e5m2_nibble_chain

FP8 E5M2 standard: BitReorderFp8E5M2 + NibbleSplit.

pyconstant

ptwm.preprocessing.CHAIN_U8_SCALE_PASSTHROUGH

python/ptwm/preprocessing/_chains.py:916
CHAIN_U8_SCALE_PASSTHROUGH: ChainBuilder = _u8_scale_passthrough_chain

Uint8 scale-block passthrough (MXFP4 E8M0 scales).

pymethod

ptwm.preprocessing.Chain.__init__

python/ptwm/preprocessing/_chains.py:1
def __init__(self, nodes: list[ChainNode], edges: list[ChainEdge], terminals: list[TerminalRef]) -> None
pyattribute

ptwm.preprocessing.Chain.edges

python/ptwm/preprocessing/_chains.py:173
edges: list[ChainEdge]
pyattribute

ptwm.preprocessing.Chain.nodes

python/ptwm/preprocessing/_chains.py:172
nodes: list[ChainNode]
pyattribute

ptwm.preprocessing.Chain.terminals

python/ptwm/preprocessing/_chains.py:174
terminals: list[TerminalRef]
pymethod

ptwm.preprocessing.Chain.to_bytes

python/ptwm/preprocessing/_chains.py:176
def to_bytes(self) -> bytes

Serialise to the wire format consumed by ``compress_model``.

The wire format is a self-contained blob consisting of a local extension table followed by table-indexed chain bytes. The conversion from legacy op-id bytes to the new format is done via the Rust bridge to ensure consistency with the Rust canonical-id implementation.
pyattribute

ptwm.preprocessing.ChainBuilder

python/ptwm/preprocessing/_chains.py:277
ChainBuilder = Callable[[list[int]], Chain | None]
pyclass

ptwm.preprocessing.ChainEdge

python/ptwm/preprocessing/_chains.py:153
class ChainEdge:
pymethod

ptwm.preprocessing.ChainEdge.__init__

python/ptwm/preprocessing/_chains.py:1
def __init__(self, src_node: int, src_output_idx: int, dst_node: int, dst_input_idx: int, role_override: bytes | None = None, vendor_bytes: bytes = b'') -> None
pyattribute

ptwm.preprocessing.ChainEdge.dst_input_idx

python/ptwm/preprocessing/_chains.py:158
dst_input_idx: int
pyattribute

ptwm.preprocessing.ChainEdge.dst_node

python/ptwm/preprocessing/_chains.py:157
dst_node: int
pyattribute

ptwm.preprocessing.ChainEdge.role_override

python/ptwm/preprocessing/_chains.py:159
role_override: bytes | None = None
pyattribute

ptwm.preprocessing.ChainEdge.src_node

python/ptwm/preprocessing/_chains.py:155
src_node: int
pyattribute

ptwm.preprocessing.ChainEdge.src_output_idx

python/ptwm/preprocessing/_chains.py:156
src_output_idx: int
pyattribute

ptwm.preprocessing.ChainEdge.vendor_bytes

python/ptwm/preprocessing/_chains.py:160
vendor_bytes: bytes = field(default=b'')
pyclass

ptwm.preprocessing.ChainNode

python/ptwm/preprocessing/_chains.py:147
class ChainNode:
pymethod

ptwm.preprocessing.ChainNode.__init__

python/ptwm/preprocessing/_chains.py:1
def __init__(self, op_id: int, params: bytes = b'') -> None
pyattribute

ptwm.preprocessing.ChainNode.op_id

python/ptwm/preprocessing/_chains.py:149
op_id: int
pyattribute

ptwm.preprocessing.ChainNode.params

python/ptwm/preprocessing/_chains.py:150
params: bytes = field(default=b'')
pyfunction

ptwm.preprocessing.chains_for

python/ptwm/preprocessing/_chains.py:1029
def chains_for(dtype_code: int, role: ClassifierRole) -> list[ChainBuilder]

Return the chain builders for ``(dtype_code, role)``.

Returns an empty list when the combination has no production entry.
pyclass

ptwm.preprocessing.ClassifierRole

python/ptwm/preprocessing/_chains.py:263
class ClassifierRole(IntEnum):

Tensor role used as the second key in :data:`PRODUCTION_CHAINS`.

pyconstant

ptwm.preprocessing.ClassifierRole.PACKED_VALUES

python/ptwm/preprocessing/_chains.py:269
PACKED_VALUES = 3
pyconstant

ptwm.preprocessing.ClassifierRole.SCALE_BLOCK

python/ptwm/preprocessing/_chains.py:267
SCALE_BLOCK = 1
pyconstant

ptwm.preprocessing.ClassifierRole.SCALE_GLOBAL

python/ptwm/preprocessing/_chains.py:268
SCALE_GLOBAL = 2
pyconstant

ptwm.preprocessing.ClassifierRole.STANDARD

python/ptwm/preprocessing/_chains.py:266
STANDARD = 0
pyfunction

ptwm.preprocessing.clear_cache

python/ptwm/preprocessing/_cache.py:164
def clear_cache() -> int

Delete every cache file. Returns the number of files removed.

pyfunction

ptwm.preprocessing.explore_chains

python/ptwm/preprocessing/_explorer.py:281
def explore_chains(dtype_code: int, role: ClassifierRole, opts: ExploreOptions | None = None) -> Iterator[ChainBuilder]

Enumerate legal chain builders for *(dtype_code, role)*.

Each yielded value is a :data:`ChainBuilder` callable ``(shape: list[int]) -> Chain``. Yielded callables also carry a ``template`` attribute (an `~ptwm.preprocessing._cache.CacheEntry`) so the user-local chain cache can record the structural template without re- running the BFS.
pyclass

ptwm.preprocessing.ExploreOptions

python/ptwm/preprocessing/_explorer.py:75
class ExploreOptions:

Search-bound configuration for the generative explorer.

pymethod

ptwm.preprocessing.ExploreOptions.__init__

python/ptwm/preprocessing/_explorer.py:1
def __init__(self, max_nodes: int = 8, max_depth: int = 5, max_candidates_per_dtype: int = 32, allow_cross_tensor: bool = False, allow_cross_plane: bool = False, time_budget_ms: int = 5000) -> None
pyattribute

ptwm.preprocessing.ExploreOptions.allow_cross_plane

python/ptwm/preprocessing/_explorer.py:83
allow_cross_plane: bool = False
pyattribute

ptwm.preprocessing.ExploreOptions.allow_cross_tensor

python/ptwm/preprocessing/_explorer.py:82
allow_cross_tensor: bool = False
pyattribute

ptwm.preprocessing.ExploreOptions.max_candidates_per_dtype

python/ptwm/preprocessing/_explorer.py:81
max_candidates_per_dtype: int = 32
pyattribute

ptwm.preprocessing.ExploreOptions.max_depth

python/ptwm/preprocessing/_explorer.py:80
max_depth: int = 5
pyattribute

ptwm.preprocessing.ExploreOptions.max_nodes

python/ptwm/preprocessing/_explorer.py:79
max_nodes: int = 8
pyattribute

ptwm.preprocessing.ExploreOptions.time_budget_ms

python/ptwm/preprocessing/_explorer.py:84
time_budget_ms: int = 5000
pyfunction

ptwm.preprocessing.load_cached_builders

python/ptwm/preprocessing/_cache.py:132
def load_cached_builders(dtype_code: int, role: ClassifierRole) -> list[ChainBuilder]

Return runtime `ChainBuilder`s for cached entries of `(dtype_code, role)`.

pyfunction

ptwm.preprocessing.load_cached_entries

python/ptwm/preprocessing/_cache.py:119
def load_cached_entries(dtype_code: int, role: ClassifierRole) -> list[CacheEntry]

Return cached entries for `(dtype_code, role)`, or `[]` on miss / corruption.

pyclass

ptwm.preprocessing.Plane

python/ptwm/preprocessing/_plane.py:8
class Plane:

One plane produced by the preprocessing pipeline.

``role`` is a free-form short string identifying the semantic content of the plane (e.g. ``"exponent"``, ``"sign_mantissa"``, ``"byte_0"``). Used by logging / diagnostics; the compression pipeline does not branch on it.
pymethod

ptwm.preprocessing.Plane.__init__

python/ptwm/preprocessing/_plane.py:1
def __init__(self, id: int, role: str, data: bytes) -> None
pyattribute

ptwm.preprocessing.Plane.data

python/ptwm/preprocessing/_plane.py:20
data: bytes
pyattribute

ptwm.preprocessing.Plane.id

python/ptwm/preprocessing/_plane.py:18
id: int
pyattribute

ptwm.preprocessing.Plane.role

python/ptwm/preprocessing/_plane.py:19
role: str
pyconstant

ptwm.preprocessing.PRODUCTION_CHAINS

python/ptwm/preprocessing/_chains.py:969
PRODUCTION_CHAINS: dict[tuple[int, ClassifierRole], list[ChainBuilder]] = {(_DT.FLOAT8_E4M3FN.code, ClassifierRole.SCALE_BLOCK): [CHAIN_F8E4M3_SCALE_PASSTHROUGH, CHAIN_F8E4M3_NIBBLE], (_DT.UINT8.code, ClassifierRole.SCALE_BLOCK): [CHAIN_U8_SCALE_PASSTHROUGH], (_DT.FLOAT32.code, ClassifierRole.SCALE_GLOBAL): [CHAIN_F32_GLOBAL_SCALE], (_DT.FLOAT4_E2M1FN_X2.code, ClassifierRole.PACKED_VALUES): [CHAIN_FP4_NIBBLE_PASSTHROUGH], (_DT.BFLOAT16.code, ClassifierRole.STANDARD): [CHAIN_BF16_SPLIT, CHAIN_BF16_SPLIT_INTDELTA, CHAIN_BF16_SPLIT_PREDICTOR_XOR, CHAIN_BF16_MICROSCALE_REPACK, CHAIN_BF16_SPLIT_MZS_K1, CHAIN_BF16_SPLIT_MZS_K2, CHAIN_BF16_SPLIT_MZS_K3, CHAIN_BF16_SPLIT_MZS_K4, CHAIN_BF16_SPHERICAL_NORMALIZE, CHAIN_BF16_ALPHA_STABLE_NORMALIZE, CHAIN_BF16_HUFF_LLM_RAW], (_DT.FLOAT16.code, ClassifierRole.STANDARD): [CHAIN_BF16_SPLIT, CHAIN_BF16_SPLIT_INTDELTA, CHAIN_BF16_SPLIT_PREDICTOR_XOR, CHAIN_BF16_SPLIT_MZS_K1, CHAIN_BF16_SPLIT_MZS_K2, CHAIN_BF16_SPLIT_MZS_K3, CHAIN_BF16_SPLIT_MZS_K4, CHAIN_FP16_SPHERICAL_NORMALIZE, CHAIN_FP16_ALPHA_STABLE_NORMALIZE, CHAIN_FP16_HUFF_LLM_RAW], (_DT.FLOAT32.code, ClassifierRole.STANDARD): [CHAIN_FP32_SPLIT, CHAIN_FP32_SPLIT_INTDELTA, CHAIN_FP32_SPLIT_PREDICTOR_XOR, CHAIN_FP32_SPLIT_MZS_K1, CHAIN_FP32_SPLIT_MZS_K2, CHAIN_FP32_SPLIT_MZS_K3, CHAIN_FP32_SPLIT_MZS_K4, CHAIN_FP32_SPHERICAL_NORMALIZE, CHAIN_FP32_ALPHA_STABLE_NORMALIZE], (_DT.FLOAT8_E4M3FN.code, ClassifierRole.STANDARD): [CHAIN_FP8E4M3_NIBBLE], (_DT.FLOAT8_E5M2.code, ClassifierRole.STANDARD): [CHAIN_FP8E5M2_NIBBLE], (_DT.UINT8.code, ClassifierRole.STANDARD): [CHAIN_BYTE_PASSTHROUGH_VALUE]}
pyfunction

ptwm.preprocessing.save_cached_entries

python/ptwm/preprocessing/_cache.py:137
def save_cached_entries(dtype_code: int, role: ClassifierRole, new_entries: Iterable[CacheEntry]) -> int

Union `new_entries` with the existing cache file. Returns count added.

Idempotent: writing the same entry twice is a no-op.
pyclass

ptwm.preprocessing.TerminalRef

python/ptwm/preprocessing/_chains.py:163
class TerminalRef:
pymethod

ptwm.preprocessing.TerminalRef.__init__

python/ptwm/preprocessing/_chains.py:1
def __init__(self, node_idx: int, output_idx: int, role: bytes) -> None
pyattribute

ptwm.preprocessing.TerminalRef.node_idx

python/ptwm/preprocessing/_chains.py:165
node_idx: int
pyattribute

ptwm.preprocessing.TerminalRef.output_idx

python/ptwm/preprocessing/_chains.py:166
output_idx: int
pyattribute

ptwm.preprocessing.TerminalRef.role

python/ptwm/preprocessing/_chains.py:167
role: bytes