{"uid_repository": {"277713878": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 277713878, "_name": {"__type__": "DynamicName", "body": "v", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": true, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "129280", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "2067311870": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 2067311870, "_name": {"__type__": "DynamicName", "body": "a", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "compression_ratio_size", "exponent": {"__type__": "DynamicName", "body": "2", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "662338339": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 662338339, "_name": {"__type__": "DynamicName", "body": "b", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": "entries_size", "exponent": null}}}, "777392202": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 777392202, "_name": {"__type__": "DynamicName", "body": "x", "subscript": null, "settings": null, "code_form": "tokens", "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "452335221": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 452335221, "_name": {"__type__": "DynamicName", "body": "n", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "streams_size", "exponent": {"__type__": "DynamicName", "body": "4", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1330421004": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1330421004, "_name": {"__type__": "DynamicName", "body": "n", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "streams", "exponent": {"__type__": "DynamicName", "body": "4", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 452335221}}, "621350211": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 621350211, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "A^{0}", "description": "Writes one at the position named by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[0] = 1", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2073802315": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 2073802315, "_name": {"__type__": "DynamicName", "body": "m", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "hidden_width_size", "exponent": {"__type__": "DynamicName", "body": "5120", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "975060836": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 975060836, "_name": {"__type__": "DynamicName", "body": "m", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "hidden_width", "exponent": {"__type__": "DynamicName", "body": "5120", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 2073802315}}, "2119517555": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2119517555, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Embedding}", "description": "Looks up the row of the embedding table named by the token identifier. The table holds one row of the hidden width for every entry of the vocabulary.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L174", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L174", "path": "inference/model.py", "line": 174, "end_line": null}], "formula": "E(t) = W[t]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "917611616": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 917611616, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Embedding}", "description": "The token identifiers are looked up in the embedding table. The lookup gives one hidden state per token. Every token of an image span carries the same placeholder identifier. The image pathway replaces the row given to the token by the embedding.", "fill_color": "#FCE0E1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L174", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L174", "path": "inference/model.py", "line": 174, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L1201", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1201", "path": "inference/model.py", "line": 1201, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L1253", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1253", "path": "inference/model.py", "line": 1253, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L62", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L62", "path": "inference/config.json", "line": 62, "end_line": null}], "formula": null, "drawing": null}}, "1636457132": {"__type__": "TapeSlot", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "TapeSlot"}, "_id": 1636457132, "_name": {"__type__": "DynamicName", "body": "\\mathrm{ids}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1384614369": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1384614369, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Model Inputs}", "description": "The model reads the token identifiers of the prompt. The identifiers are read again far inside the model, because Engram hashes them before layers 1 and 14. A sublayer of the model reads the hidden state alone. The identifiers therefore travel on the tape. This box writes them to their slot once. The embedding reads a second copy. The released code computes the hashes of Engram once, before the first layer.", "fill_color": "#FCE0E1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L1241-L1243", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1241-L1243", "path": "inference/model.py", "line": 1241, "end_line": 1243}, {"__type__": "CodeReference", "label": "inference/model.py L1249-L1252", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1249-L1252", "path": "inference/model.py", "line": 1249, "end_line": 1252}, {"__type__": "CodeReference", "label": "inference/model.py L1267", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1267", "path": "inference/model.py", "line": 1267, "end_line": null}], "formula": null, "drawing": null}}, "242910120": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 242910120, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1391415111": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1391415111, "_name": {"__type__": "DynamicName", "body": "\\varepsilon", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1275413849": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1275413849, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1711934402": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1711934402, "_name": {"__type__": "DynamicName", "body": "\\varepsilon_{\\mathrm{hc}}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1349575359": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1349575359, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "109908815": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 109908815, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2030479332": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2030479332, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "579469646": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 579469646, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{First Sinkhorn Round}", "description": "The first round of Sinkhorn normalisation differs from the nineteen after it. Every entry of the combine matrix is replaced by its exponential. Each row is divided by its sum. The exponential and the division together are a softmax along the row. The released kernel then adds a small epsilon to every entry. The exponential of a large negative number underflows to zero. With the epsilon such an entry is still above zero. Each column is then divided by its sum plus the same epsilon.", "fill_color": "#F4EEFB", "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L436-L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L436-L443", "path": "inference/kernel.py", "line": 436, "end_line": 443}, {"__type__": "CodeReference", "label": "inference/kernel.py L445-L448", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L445-L448", "path": "inference/kernel.py", "line": 445, "end_line": 448}, {"__type__": "CodeReference", "label": "inference/config.json L42", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L42", "path": "inference/config.json", "line": 42, "end_line": null}], "formula": null, "drawing": null}}, "321235664": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 321235664, "_name": null}, "repetition": {"__type__": "Integer", "_value": 19}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Sinkhorn Round}", "description": "Each of the nineteen later rounds of Sinkhorn normalisation divides every row of the combine matrix by its sum plus a small epsilon. Each round then divides every column by its sum plus the same epsilon. A row or a column that sums to zero would otherwise be divided by zero. The epsilon keeps the division defined. After the rounds every row and every column sums to about one. A matrix with those sums is doubly stochastic. Each new stream is therefore a weighted average of the four old streams. The sum of the streams is kept.", "fill_color": "#F4EEFB", "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L450-L458", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L450-L458", "path": "inference/kernel.py", "line": 450, "end_line": 458}, {"__type__": "CodeReference", "label": "inference/config.json L41", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L41", "path": "inference/config.json", "line": 41, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L42", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L42", "path": "inference/config.json", "line": 42, "end_line": null}], "formula": null, "drawing": null}}, "308478123": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 308478123, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Mixing Coefficients}", "description": "This block predicts the mixing coefficients of one token. The four streams are normalised together. Three linear maps read the normalised streams. The first gives the collapse vector through a sigmoid plus a small epsilon, so the weight of a stream is above zero. The second gives the output vector through a doubled sigmoid. The third gives the four-by-four combine matrix. Twenty rounds of Sinkhorn normalisation make that matrix nearly doubly stochastic. Every row and every column of that matrix sums to about one.", "fill_color": "#E8DFF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L948-L955", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L948-L955", "path": "inference/model.py", "line": 948, "end_line": 955}, {"__type__": "CodeReference", "label": "inference/model.py L941-L946", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L941-L946", "path": "inference/model.py", "line": 941, "end_line": 946}, {"__type__": "CodeReference", "label": "inference/kernel.py L426-L431", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L426-L431", "path": "inference/kernel.py", "line": 426, "end_line": 431}], "formula": null, "drawing": null}}, "999722612": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 999722612, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2095719082": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2095719082, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "758490154": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 758490154, "_name": {"__type__": "DynamicName", "body": "\\epsilon", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1021115887": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1021115887, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2130048138": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 2130048138, "_name": {"__type__": "DynamicName", "body": "h", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "heads_size", "exponent": {"__type__": "DynamicName", "body": "64", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "706534913": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 706534913, "_name": {"__type__": "DynamicName", "body": "h", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "heads", "exponent": {"__type__": "DynamicName", "body": "64", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 2130048138}}, "1433843082": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1433843082, "_name": {"__type__": "DynamicName", "body": "q", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "query_rank_size", "exponent": {"__type__": "DynamicName", "body": "1280", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1128010231": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1128010231, "_name": {"__type__": "DynamicName", "body": "q", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "query_rank", "exponent": {"__type__": "DynamicName", "body": "1280", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1433843082}}, "1351821723": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1351821723, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "296734870": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 296734870, "_name": {"__type__": "DynamicName", "body": "c", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "latent_width_size", "exponent": {"__type__": "DynamicName", "body": "512", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1056277277": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1056277277, "_name": {"__type__": "DynamicName", "body": "c", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "latent_width", "exponent": {"__type__": "DynamicName", "body": "512", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 296734870}}, "1181195560": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1181195560, "_name": {"__type__": "DynamicName", "body": "t", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "32", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "182489280": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 182489280, "_name": {"__type__": "DynamicName", "body": "\\bar{z}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "448", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__type__": "Addition", "content": [{"__ref__": 296734870}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -2}, {"__ref__": 1181195560}]}]}}, "1821531651": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1821531651, "_name": {"__type__": "DynamicName", "body": "z", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "64", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": 2}, {"__ref__": 1181195560}]}}, "35354379": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 35354379, "_name": {"__type__": "DynamicName", "body": "t", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "32", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1181195560}}, "29130293": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 29130293, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1000427244": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1000427244, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1477143043": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1477143043, "_name": {"__type__": "DynamicName", "body": "\\beta_{0}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1381921121": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1381921121, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "356046016": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 356046016, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "852155779": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 852155779, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "923799071": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 923799071, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Rotary Embedding}", "description": "Each vector is rotated by the position of its token. The operation is called a rotary embedding. The last 64 of the 512 channels of each vector are taken two at a time as points in a plane. The rotation turns each point about the origin. The angle of the turn grows with the position of the token. Each pair of channels turns at a speed of its own. A query and a key that are both turned score against each other by how far apart their tokens are, because the product of two turned points depends on the difference of the two angles. The other 448 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels. The released code rotates the query of every head and the window latent of every token this way. The window latent is both the key and the value of its token.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L392-L406", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L406", "path": "inference/model.py", "line": 392, "end_line": 406}, {"__type__": "CodeReference", "label": "inference/model.py L771-L772", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L771-L772", "path": "inference/model.py", "line": 771, "end_line": 772}, {"__type__": "CodeReference", "label": "inference/model.py L705-L706", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L705-L706", "path": "inference/model.py", "line": 705, "end_line": 706}, {"__type__": "CodeReference", "label": "inference/model.py L368-L389", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L368-L389", "path": "inference/model.py", "line": 368, "end_line": 389}, {"__type__": "CodeReference", "label": "inference/model.py L680-L687", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L680-L687", "path": "inference/model.py", "line": 680, "end_line": 687}], "formula": "\\begin{gathered} y[i_{x}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{x}, |\\bar{z}| + 2 i_{t} + 1] = F[i_{x}, i_{t}]\\, \\big(v[i_{x}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{x}, |\\bar{z}| + 2 i_{t} + 1]\\big) \\\\ y[i_{x}, i_{\\bar{z}}] = v[i_{x}, i_{\\bar{z}}] \\end{gathered}", "drawing": null}}, "1811534717": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1811534717, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1388372297": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1388372297, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "30666566": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 30666566, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1559875282": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1559875282, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1375795238": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1375795238, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "559632892": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 559632892, "_name": {"__type__": "DynamicName", "body": "\\tilde{E}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "window_scale_groups_size", "exponent": {"__type__": "DynamicName", "body": "16", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "835835122": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 835835122, "_name": {"__type__": "DynamicName", "body": "\\tilde{E}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "window_scale_groups", "exponent": {"__type__": "DynamicName", "body": "16", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 559632892}}, "903385593": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 903385593, "_name": {"__type__": "DynamicName", "body": "\\hat{y}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "channels_per_power_of_two_scale_size", "exponent": {"__type__": "DynamicName", "body": "32", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1981822304": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1981822304, "_name": {"__type__": "DynamicName", "body": "\\hat{y}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "channels_per_power_of_two_scale", "exponent": {"__type__": "DynamicName", "body": "32", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 903385593}}, "72447950": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 72447950, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "85460599": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 85460599, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "693463748": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 693463748, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1002981784": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1002981784, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "899603404": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 899603404, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2069837421": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2069837421, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1652884106": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1652884106, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1058467236": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1058467236, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1031685272": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1031685272, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2094582236": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2094582236, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1257516800": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1257516800, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1446818017": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1446818017, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "207717483": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 207717483, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{FP8 Round Trip of a Window Latent}", "description": "The box rounds one token's window latent the way the released window cache holds it. The channels are cut into groups of 32. Each group is divided by a scale of its own. Every channel is rounded to the nearest number of the eight-bit E4M3 form. The group is multiplied by its scale again. The largest E4M3 number is 448. The scale of a group is its largest magnitude divided by 448 and rounded up to a power of two. The scale is therefore stored as an exponent alone, which is the UE8M0 form. A logarithm to base two, a ceiling and a power of two state the rounding of the scale. The ceiling is a generic operator because the package has no numeric for it. The window cache holds the latents of the newest 128 tokens in every layer. The released report says the window cache stays at eight bits because the window latents are sensitive to quantisation. The released code rounds a window latent in place after it has rotated the latent.", "fill_color": "#DDE6F2", "references": [{"__type__": "CodeReference", "label": "inference/model.py L705-L707", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L705-L707", "path": "inference/model.py", "line": 705, "end_line": 707}, {"__type__": "CodeReference", "label": "inference/model.py L27-L30", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L27-L30", "path": "inference/model.py", "line": 27, "end_line": 30}, {"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L74-L80", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74-L80", "path": "inference/kernel.py", "line": 74, "end_line": 80}, {"__type__": "CodeReference", "label": "inference/kernel.py L22-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L37", "path": "inference/kernel.py", "line": 22, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}], "formula": "\\begin{gathered} a[i_{\\tilde{E}}] = \\max_{i_{\\hat{y}} \\in \\hat{y}} \\lvert v[\\lvert \\hat{y} \\rvert\\, i_{\\tilde{E}} + i_{\\hat{y}}] \\rvert \\\\ s[i_{\\tilde{E}}] = 2^{\\lceil \\log_{2} (\\max(a[i_{\\tilde{E}}],\\, 10^{-4}) / 448) \\rceil} \\\\ \\hat{v}[\\lvert \\hat{y} \\rvert\\, i_{\\tilde{E}} + i_{\\hat{y}}] = s[i_{\\tilde{E}}]\\, \\mathrm{E4M3}\\big(\\ulcorner v[\\lvert \\hat{y} \\rvert\\, i_{\\tilde{E}} + i_{\\hat{y}}] / s[i_{\\tilde{E}}] \\lrcorner_{-448}^{448}\\big) \\end{gathered}", "drawing": null}}, "990851622": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 990851622, "_name": {"__type__": "DynamicName", "body": "w", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "window_size", "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1922576489": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1922576489, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "95011573": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 95011573, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "86086985": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 86086985, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1845635261": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1845635261, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1043465010": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1043465010, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1392885062": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1392885062, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1278767244": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1278767244, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots and the learned sink. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "1703485446": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1703485446, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1601504201": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1601504201, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1907365617": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1907365617, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "617321454": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 617321454, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1215894946": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1215894946, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "376895794": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 376895794, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Inverse Rotary Embedding}", "description": "The last 64 of the 512 channels of each attention output are taken two at a time as points in a plane. Each point is turned back by the angle of the position of its query. The output of a query is a weighted sum of latents. Each latent in the sum was turned by the angle of its own position. That position is the token for a window latent or a decoder entry, and the first token of its group for an encoder entry. Each latent in the sum is then left turned by the difference between that position and the position of the query, so the output carries no position counted from the start of the context. The other 448 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels. The map after the circle takes the conjugate of every turn, which is the turn through the same angle in the other direction.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L780-L781", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780-L781", "path": "inference/model.py", "line": 780, "end_line": 781}, {"__type__": "CodeReference", "label": "inference/model.py L392-L395", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L395", "path": "inference/model.py", "line": 392, "end_line": 395}, {"__type__": "CodeReference", "label": "inference/model.py L398-L399", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L398-L399", "path": "inference/model.py", "line": 398, "end_line": 399}], "formula": "\\begin{gathered} y[i_{x}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{x}, |\\bar{z}| + 2 i_{t} + 1] = F[i_{x}, i_{t}]\\, \\big(v[i_{x}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{x}, |\\bar{z}| + 2 i_{t} + 1]\\big) \\\\ y[i_{x}, i_{\\bar{z}}] = v[i_{x}, i_{\\bar{z}}] \\end{gathered}", "drawing": null}}, "1338770949": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1338770949, "_name": {"__type__": "DynamicName", "body": "g", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "head_groups_size", "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "13639104": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 13639104, "_name": {"__type__": "DynamicName", "body": "g", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "head_groups", "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1338770949}}, "141018594": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 141018594, "_name": {"__type__": "DynamicName", "body": "j", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "heads_per_group_size", "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "182848503": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 182848503, "_name": {"__type__": "DynamicName", "body": "j", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "heads_per_group", "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 141018594}}, "792264637": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 792264637, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1733334145": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1733334145, "_name": {"__type__": "DynamicName", "body": "o", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "output_rank_size", "exponent": {"__type__": "DynamicName", "body": "1024", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "55955818": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 55955818, "_name": {"__type__": "DynamicName", "body": "o", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "output_rank", "exponent": {"__type__": "DynamicName", "body": "1024", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1733334145}}, "1551896111": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1551896111, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1309719187": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1309719187, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Grouped Output Projection}", "description": "The projection maps one query's 64 heads into the residual width. The heads are cut into eight groups of eight. Each group's values are mapped to its own 1024-wide rank by a weight of its own. One final weight reads all eight ranks together.", "fill_color": "#D5C8E8", "references": [{"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": null, "drawing": null}}, "517862890": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 517862890, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Sliding Window Attention}", "description": "Each of the first two layers attends over the last 128 tokens alone. The layer computes its query and one window latent per token from its own hidden state. The window latent is both the key and the value of its token. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. These two layers rotate with the base 10000. Every layer from 2 to 39 rotates with the base 160000 and YaRN, which slows the slowest rotations for contexts longer than the trained one. The softmax also holds a learned sink, which is one number per head that lets the head give part of its weight to no token. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection.", "fill_color": "#DFF0D8", "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L680-L687", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L680-L687", "path": "inference/model.py", "line": 680, "end_line": 687}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}], "formula": null, "drawing": null}}, "407386606": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 407386606, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "807017432": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 807017432, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "937953296": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 937953296, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "36706552": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 36706552, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "411460338": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 411460338, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1043665725": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1043665725, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "504897397": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 504897397, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1875313949": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1875313949, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "836795547": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 836795547, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1086684913": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1086684913, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "85202227": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 85202227, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1621082764": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1621082764, "_name": {"__type__": "DynamicName", "body": "e", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "experts_size", "exponent": {"__type__": "DynamicName", "body": "384", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "401190795": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 401190795, "_name": {"__type__": "DynamicName", "body": "e", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "experts", "exponent": {"__type__": "DynamicName", "body": "384", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1621082764}}, "1627256985": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1627256985, "_name": {"__type__": "DynamicName", "body": "\\tau", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "182144126": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 182144126, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "636446273": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 636446273, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1323411805": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1323411805, "_name": {"__type__": "DynamicName", "body": "k", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "6", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1137802141": {"__type__": "SparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "SparseAxis"}, "_id": 1137802141, "_name": {"__type__": "DynamicName", "body": "k/e", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "384", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1621082764}, "activity": {"__ref__": 1323411805}}, "296967158": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 296967158, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1615673735": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1615673735, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1214936271": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1214936271, "_name": {"__type__": "DynamicName", "body": "\\varepsilon_{\\mathrm{r}}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1182676660": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1182676660, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1189046433": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1189046433, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Expert Gate}", "description": "The gate box computes the gates of one token over the routed experts. The router scores every expert. The scores are divided by a temperature before the score function. A temperature of one leaves the scores unchanged. The released configuration sets none, so the released value is one. A correction bias of one learned number per expert is added to one copy of the scores. The six experts are chosen from that copy. The released model holds a second bias for a token inside an image span. A model that reads text alone never reaches that bias. A chosen slot carries one where the biased score of its expert is above zero and zero elsewhere. The score without the bias is multiplied by that number. The score function returns a value above zero. The number is therefore one unless the bias of a chosen expert is at or below minus its score. The released code reads the score without the bias at every chosen expert whatever the sign of the biased score. The expression and the released code therefore differ in that one case, where the expression gives the expert a gate of zero. The six gates are divided by their sum plus a small epsilon and scaled by the route scale. The epsilon keeps the division defined when all six scores have underflowed to zero.", "fill_color": "#F7E0EF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L809-L827", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L809-L827", "path": "inference/model.py", "line": 809, "end_line": 827}, {"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L818-L820", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L818-L820", "path": "inference/model.py", "line": 818, "end_line": 820}, {"__type__": "CodeReference", "label": "inference/model.py L825", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L825", "path": "inference/model.py", "line": 825, "end_line": null}], "formula": null, "drawing": null}}, "2129965811": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2129965811, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1981447839": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1981447839, "_name": {"__type__": "DynamicName", "body": "f", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "expert_width_size", "exponent": {"__type__": "DynamicName", "body": "2304", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "435880113": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 435880113, "_name": {"__type__": "DynamicName", "body": "f", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "expert_width", "exponent": {"__type__": "DynamicName", "body": "2304", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1981447839}}, "432449881": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 432449881, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "96553797": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 96553797, "_name": {"__type__": "DynamicName", "body": "\\lambda", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "671146010": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 671146010, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2070052423": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2070052423, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "916785349": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 916785349, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "164042319": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 164042319, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1471042596": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1471042596, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "511925873": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 511925873, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1083357423": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1083357423, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Routed Experts}", "description": "The routed experts run on one token. A gate projection is limited from above at the SwiGLU limit and passes through the SiLU activation. An up projection is limited to the range from minus the limit to the limit. The two are multiplied. A down projection returns the product to the residual width. The diagonal keeps the expert chosen by each of the six slots. The two limits come from training, where they keep the activations inside the range represented by the FP8 and FP4 number formats. The SiLU is close to zero at every large negative input, so the gate branch is bounded on that side without a limit.", "fill_color": "#C1E8F7", "references": [{"__type__": "CodeReference", "label": "inference/model.py L841-L851", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L841-L851", "path": "inference/model.py", "line": 841, "end_line": 851}, {"__type__": "CodeReference", "label": "inference/model.py L845-L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L845-L847", "path": "inference/model.py", "line": 845, "end_line": 847}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": null, "drawing": null}}, "1379230893": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1379230893, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1003648250": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1003648250, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "131843983": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 131843983, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1683887822": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1683887822, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1409242776": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1409242776, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1511701545": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1511701545, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2123592162": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2123592162, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1732281219": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1732281219, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "461059629": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 461059629, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "55829965": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 55829965, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Shared Expert}", "description": "Every token runs the shared expert, which has the expert width of a routed expert and no router. A gate projection is limited from above at the SwiGLU limit and passes through the SiLU activation. An up projection is limited on both sides. The two are multiplied. A down projection returns the product to the residual width. The limits are the ones applied by the routed experts. The limits keep the activations inside the range represented by the FP8 and FP4 number formats.", "fill_color": "#DFF0D8", "references": [{"__type__": "CodeReference", "label": "inference/model.py L887", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L887", "path": "inference/model.py", "line": 887, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L845-L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L845-L847", "path": "inference/model.py", "line": 845, "end_line": 847}, {"__type__": "CodeReference", "label": "inference/model.py L903", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L903", "path": "inference/model.py", "line": 903, "end_line": null}], "formula": null, "drawing": null}}, "811098828": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 811098828, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1418510891": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1418510891, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1758278491": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1758278491, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Mixture of Experts}", "description": "One token passes through the mixture. The gate box reads the hidden state of the token, chooses six experts and weights them. The outputs of the six are summed under their gates. The output of the shared expert is added.", "fill_color": "#FFE2BB", "references": [{"__type__": "CodeReference", "label": "inference/model.py L889-L904", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L889-L904", "path": "inference/model.py", "line": 889, "end_line": 904}], "formula": null, "drawing": null}}, "1582799004": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1582799004, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1556924115": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1556924115, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "821503956": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 821503956, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "1789063455": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1789063455, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Sliding Window Layer}", "description": "Layer 0, one of the two sliding-window layers: a sliding-window attention sublayer and a mixture sublayer, each inside the four-stream residual. The released model has two such layers. They are drawn one by one because Engram is applied to the residual between them. The two layers are therefore not two iterations of one repeated block.", "fill_color": "#EFEFF4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L26", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L26", "path": "inference/config.json", "line": 26, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L52", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L52", "path": "inference/config.json", "line": 52, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L1262-L1263", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1262-L1263", "path": "inference/model.py", "line": 1262, "end_line": 1263}], "formula": null, "drawing": null}}, "1779995924": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1779995924, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "142403013": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 142403013, "_name": {"__type__": "DynamicName", "body": "\\hat{v}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "99092", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "5265176": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 5265176, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Token Map}", "description": "Replaces a token identifier by the identifier of the token's normalised text, so that tokens such as The, the and THE hash alike. The table M is fixed. The released code builds it once from the tokenizer. Training never changes it. The result of the table is a natural number below 99,092.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L17-L61", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L17-L61", "path": "inference/engram.py", "line": 17, "end_line": 61}, {"__type__": "CodeReference", "label": "inference/engram.py L164", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L164", "path": "inference/engram.py", "line": 164, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L48", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L48", "path": "inference/config.json", "line": 48, "end_line": null}], "formula": "\\mathrm{cmp}(\\mathrm{id})[i_{x}] = M[\\mathrm{id}[i_{x}]]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "188737552": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 188737552, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Token Map}", "description": "Each token identifier is replaced by the identifier of its normalised text. The normalisation lowercases the text, removes accents and joins runs of white space. The tokens The, the and THE therefore receive one identifier. The n-grams that hold them address the same rows. The map is a lookup in a table M that has one entry for every token identifier and returns one of 99,092 compressed identifiers. The table is fixed. The released code builds it once from the tokenizer. Training never changes it.", "fill_color": "#FCE0E1", "references": [{"__type__": "CodeReference", "label": "inference/engram.py L17-L61", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L17-L61", "path": "inference/engram.py", "line": 17, "end_line": 61}, {"__type__": "CodeReference", "label": "inference/engram.py L164", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L164", "path": "inference/engram.py", "line": 164, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L48", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L48", "path": "inference/config.json", "line": 48, "end_line": null}], "formula": "\\mathrm{cmp}(\\mathrm{id})[i_{x}] = M[\\mathrm{id}[i_{x}]]", "drawing": null}}, "1428246250": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1428246250, "_name": {"__type__": "DynamicName", "body": "L", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "4", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "943552329": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 943552329, "_name": {"__type__": "DynamicName", "body": "L|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "4", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1428246250}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "1906749507": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1906749507, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "ngram", "description": "The view is the lookback of Engram. Slot i_L of token i_x reads the token i_L places before it. Slot 0 is therefore the token itself. The four slots hold the longest n-gram. A slot that reaches before the first token holds no value. The guarded axis L|x records which slots hold no value. The same view reads the compressed identifiers and the modality.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L171-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L171-L172", "path": "inference/engram.py", "line": 171, "end_line": 172}], "formula": "y[i_{x}, i_{L|x}] = x[i_{x} - i_{L|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1390890971": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1390890971, "_name": {"__type__": "DynamicName", "body": "\\bar{p}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1178248995": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1178248995, "_name": {"__type__": "DynamicName", "body": "G", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "3", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1396444206": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1396444206, "_name": {"__type__": "DynamicName", "body": "K", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "912013943": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 912013943, "_name": {"__type__": "DynamicName", "body": "GK", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "24", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 1178248995}, {"__ref__": 1396444206}]}}, "70620287": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 70620287, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "p_{1}", "description": "The primes of the hash of layer 1 are one for each of the |G||K| pairs of an n-gram order and a hash head, in the order the released layout draws them: the successive primes above the row count given by the configuration for one pair, sixteen million. The datatype of the array is the naturals below the bound p-bar, which is one bound standing over every prime of the layer, because a wire carries one datatype. A remainder after division by a prime is below that prime, so the same bound holds every remainder as well. The table of layer 1 has 384,006,168 rows, which is the sum of its primes. The offset of a pair is the sum of the primes before it. The row ranges of the pairs are therefore disjoint.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L102-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L102-L118", "path": "inference/engram.py", "line": 102, "end_line": 118}], "formula": "p_{1}[0] < p_{1}[1] < \\dots < p_{1}[|G||K| - 1] < \\bar{p}, \\quad p_{1}[i_{GK}] \\text{ prime}, \\quad T_{1} = \\sum_{i_{GK} \\in GK} p_{1}[i_{GK}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2031763661": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2031763661, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\alpha_{1}", "description": "The multipliers of the hash of layer 1 are one odd integer per lookback slot. They are drawn once for that layer from a random number generator seeded by the layer, so the two Engram layers hash differently. An identifier is below the identifier bound. A multiplier is below 2^63 divided by that bound. The product of the two is therefore below 2^63 and is a 64-bit integer with nothing cast.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L64-L83", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L64-L83", "path": "inference/engram.py", "line": 64, "end_line": 83}], "formula": "\\alpha_{1}[i_{L}] < 2^{63} / \\hat{v}, \\quad \\alpha_{1}[i_{L}] \\text{ odd}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1261606321": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1261606321, "_name": {"__type__": "DynamicName", "body": "G", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "3", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1178248995}}, "1379823832": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1379823832, "_name": {"__type__": "DynamicName", "body": "L'|G", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "4", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1428246250}, "guides": [{"__ref__": 1261606321}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": 1}, "shift": {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 1178248995}]}]}, "extent": null}, "1943498271": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1943498271, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "prefix", "description": "The view is the prefix of the lookback slots read by each n-gram order. The order i_G reads the slot i_L' + i_G + 1 - |G| of the products. The order therefore covers the newest i_G + 2 slots. The slots not reached by the order hold no value. The exclusive or that follows ignores those slots, so one reduction gives the hash of the 2-gram, the 3-gram and the 4-gram.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L179-L184", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L179-L184", "path": "inference/engram.py", "line": 179, "end_line": 184}], "formula": "y[i_{G}, i_{L'|G}] = x[i_{G} + i_{L'|G} + 1 - |G|_{3}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1345757761": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1345757761, "_name": {"__type__": "DynamicName", "body": "K", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1396444206}}, "270461316": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 270461316, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pair", "description": "The view reads an array over the |G||K| pairs as an array over the n-gram orders and the hash heads. The pair |K| i_G + i_K stands at the order i_G and the head i_K. The released layout draws the primes in that order. The primes and the offsets are both read this way.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L108-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L108-L118", "path": "inference/engram.py", "line": 108, "end_line": 118}], "formula": "y[i_{G}, i_{K}] = x[|K|_{8}\\, i_{G} + i_{K}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "334943757": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 334943757, "_name": {"__type__": "DynamicName", "body": "GK'|GK", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "24", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 1178248995}, {"__ref__": 1396444206}]}, "guides": [{"__ref__": 912013943}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": 1}, "shift": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 1178248995}, {"__ref__": 1396444206}]}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 1178248995}, {"__ref__": 1396444206}]}}, "1483777515": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1483777515, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "before", "description": "The view reads the primes that stand before each pair of an n-gram order and a hash head. The pair i_GK reads the prime at i_GK' + i_GK - |GK|. The primes before the pair therefore fill the last i_GK slots. The earlier slots hold no value. The sum over the slots is the offset of the pair. The rows of that pair begin at the offset in the table.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L108-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L108-L118", "path": "inference/engram.py", "line": 108, "end_line": 118}], "formula": "y[i_{GK}, i_{GK'|GK}] = x[i_{GK} + i_{GK'|GK} - |G|_{3} |K|_{8}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "336459836": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 336459836, "_name": {"__type__": "DynamicName", "body": "T", "subscript": {"__type__": "DynamicName", "body": "1", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "384006168", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1468796888": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1468796888, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pair", "description": "The view reads an array over the |G||K| pairs as an array over the n-gram orders and the hash heads. The pair |K| i_G + i_K stands at the order i_G and the head i_K. The released layout draws the primes in that order. The primes and the offsets are both read this way.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L108-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L108-L118", "path": "inference/engram.py", "line": 108, "end_line": 118}], "formula": "y[i_{G}, i_{K}] = x[|K|_{8}\\, i_{G} + i_{K}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "749433221": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 749433221, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\mathrm{cast}", "description": "Pyncd represents Natural values as a bounded datatype. Here the cast names the correct size for the subsequent embedding, which uses the bound to determine its size. The value read and the value written are both held in INT64.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L184", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L184", "path": "inference/engram.py", "line": 184, "end_line": null}], "formula": "y = x, \\quad \\mathrm{Nat}(T_1:384006168 + \\bar{p}) \\to \\mathrm{Nat}(T_1:384006168)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1789295170": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1789295170, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{N-gram Hash}", "description": "An n-gram here is the run of the last two, three or four compressed identifiers that ends at a token. A table with one row for every possible n-gram would be too large, because 99,092 identifiers give about 10^20 different runs of four. Each n-gram is therefore hashed into one of about sixteen million rows per order and head. The hash is written out over integer operators. Each of the |L| identifiers is multiplied by the fixed odd multiplier of its slot. A multiplier is below 2^63 divided by the identifier bound, so every product is below 2^63 and is a 64-bit integer. The view named prefix reads, for each order, the products of the newest two, three or four slots. A slot not reached by the order holds the unit. The bitwise exclusive or of those products is one 64-bit integer per order. The exclusive or is defined on bit patterns, so the products carry a bound that is a power of two. The remainder of that integer after division by the prime of the order and the head is below that prime. The primes are one fixed array over the |G||K| pairs of an order and a head. The array holds the successive primes above 16,000,000 in the order the released layout draws them. The offset of a pair is the sum of the primes before it. The primes before a pair are read through the view named before and summed. The remainder plus the offset is therefore a row number in a range used by no other pair. The table has as many rows as the primes sum to. The primes and the offsets are then read over the orders and the heads. Two n-grams may share a row under one head and seldom share one under another, because the eight heads divide by eight different primes. The multipliers, the primes and the offsets belong to layer 1. The table addressed by that layer holds 384,006,168 rows, which is the sum of its 24 primes.", "fill_color": "#F5E6D9", "references": [{"__type__": "CodeReference", "label": "inference/engram.py L169-L175", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L169-L175", "path": "inference/engram.py", "line": 169, "end_line": 175}, {"__type__": "CodeReference", "label": "inference/engram.py L179-L184", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L179-L184", "path": "inference/engram.py", "line": 179, "end_line": 184}, {"__type__": "CodeReference", "label": "inference/engram.py L64-L83", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L64-L83", "path": "inference/engram.py", "line": 64, "end_line": 83}, {"__type__": "CodeReference", "label": "inference/engram.py L108-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L108-L118", "path": "inference/engram.py", "line": 108, "end_line": 118}, {"__type__": "CodeReference", "label": "inference/engram.py L148-L149", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L148-L149", "path": "inference/engram.py", "line": 148, "end_line": 149}], "formula": "\\mathrm{hash}_{1}(c)[i_{G}, i_{K}] = \\Big( \\bigoplus_{i_{L} \\in L,\\; i_{L} \\leq i_{G} + 1} \\alpha_{1}[i_{L}]\\, c[i_{L}] \\Big) \\bmod p_{1}[i_{G}, i_{K}] + o_{1}[i_{G}, i_{K}]", "drawing": null}}, "425368813": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 425368813, "_name": {"__type__": "DynamicName", "body": "D", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "256", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "745749420": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 745749420, "_name": {"__type__": "DynamicName", "body": "D", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "256", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 425368813}}, "43558003": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 43558003, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{N-gram Table}", "description": "Returns the learned row of width 256 named by a row number, from the n-gram table of layer 1, which holds 384,006,168 rows. One token looks up 24 rows, one for every n-gram order and hash head. The released code stores the rows in FP8 with one scale for every 32 channels and converts a row to reals when the row is read.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L296-L325", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L296-L325", "path": "inference/model.py", "line": 296, "end_line": 325}, {"__type__": "CodeReference", "label": "inference/config.json L45", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L45", "path": "inference/config.json", "line": 45, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L64", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L64", "path": "inference/config.json", "line": 64, "end_line": null}], "formula": "E_{1}(r)[i_{D}] = N_{1}[r, i_{D}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "978840573": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 978840573, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{N-gram Table}", "description": "Each of the 24 row numbers of a token, one for every n-gram order and hash head, is looked up in the learned table of this layer, which gives 24 vectors of width |D| = 256. One linear map reads the 24 vectors together and gives one key for each residual stream. A second map gives one value shared by every stream. The released code holds the two maps as one weight and cuts its result in two. The table of layer 1 has 384,006,168 rows, which is the sum of the 24 primes of the layer. The released code stores the rows in FP8 with one scale for every 32 channels and converts a row to reals when the row is read.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L309-L310", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L309-L310", "path": "inference/model.py", "line": 309, "end_line": 310}, {"__type__": "CodeReference", "label": "inference/model.py L317-L321", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L317-L321", "path": "inference/model.py", "line": 317, "end_line": 321}, {"__type__": "CodeReference", "label": "inference/model.py L343-L345", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L343-L345", "path": "inference/model.py", "line": 343, "end_line": 345}, {"__type__": "CodeReference", "label": "inference/model.py L353-L355", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L353-L355", "path": "inference/model.py", "line": 353, "end_line": 355}, {"__type__": "CodeReference", "label": "inference/config.json L45", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L45", "path": "inference/config.json", "line": 45, "end_line": null}], "formula": null, "drawing": null}}, "1601218126": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1601218126, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1037965390": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1037965390, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1455641400": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1455641400, "_name": {"__type__": "DynamicName", "body": "\\varepsilon_{\\mathrm{g}}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1876072715": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1876072715, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\mathrm{gate}", "description": "This map is the gate of the Engram module. The normalised dot product of a stream with its key is divided by the square root of the width. The magnitude of that product is kept at or above one millionth. The square root of the magnitude is taken. The sign of the dot product is put back. A sigmoid follows. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L341", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L341", "path": "inference/model.py", "line": 341, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L360-L362", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L360-L362", "path": "inference/model.py", "line": 360, "end_line": 362}], "formula": "y = \\sigma(\\operatorname{sign}(x |m|_{5120}^{-2^{-1}}) \\sqrt{\\max(\\lvert x |m|_{5120}^{-2^{-1}} \\rvert, \\varepsilon_{\\mathrm{g}})})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1172950056": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1172950056, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Engram Gate}", "description": "A stream receives the looked-up value multiplied by its gate. Each stream of a token is multiplied channel by channel with its key and with a learned weight. The products are summed over the channels. The sum is divided by the root mean square of the stream, by the root mean square of the key and by the square root of the width. Multiplying the stream or the key by a positive constant therefore leaves the result unchanged. The two root mean squares have an epsilon under the root and no learned gain. The gate is the sigmoid of the signed square root of that number. The gate is above one half where the sum is positive and below one half where the sum is negative. The released code multiplies the gate by one minus the modality of the token. The multiplication sets the gate to zero at an image token. A model that reads text alone holds no image token.", "fill_color": "#F7E0EF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L347-L348", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L347-L348", "path": "inference/model.py", "line": 347, "end_line": 348}, {"__type__": "CodeReference", "label": "inference/model.py L356-L362", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L356-L362", "path": "inference/model.py", "line": 356, "end_line": 362}, {"__type__": "CodeReference", "label": "inference/model.py L363-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L363-L364", "path": "inference/model.py", "line": 363, "end_line": 364}], "formula": "y[i_{x}, i_{n}] = \\frac{\\sum_{i_{m} \\in m} X[i_{x}, i_{n}, i_{m}]\\, w[i_{n}, i_{m}]\\, k[i_{x}, i_{n}, i_{m}]}{\\mathrm{rms}(X[i_{x}, i_{n}])\\, \\mathrm{rms}(k[i_{x}, i_{n}])}, \\quad g[i_{x}, i_{n}] = \\sigma\\Big( \\mathrm{sign}(y) \\sqrt{\\max(\\lvert y \\rvert \\lvert m \\rvert^{-1/2}, \\varepsilon_{\\mathrm{g}})} \\Big)", "drawing": null}}, "1817300803": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1817300803, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1509955038": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1509955038, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "77275471": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 77275471, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Engram of Layer 1}", "description": "Engram is a memory. The model reads the memory by looking up short runs of text. An n-gram here is the run of the last two, three or four token identifiers that ends at a token. The three n-grams of each token are hashed into row numbers of a large learned table. The rows are read. The rows are projected into one key for each of the four residual streams and one value shared by the streams. Each stream is compared with its key. The comparison gives a gate g between zero and one. The value v times the gate is added to the stream. The module reads the token identifiers from the tape slot written by the input of the model. The module runs on the residual that enters layer 1. Layers 1 and 14 each hold a module with a table and weights of its own.", "fill_color": "#E4EFD9", "references": [{"__type__": "CodeReference", "label": "inference/model.py L328-L365", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L328-L365", "path": "inference/model.py", "line": 328, "end_line": 365}, {"__type__": "CodeReference", "label": "inference/model.py L1249-L1252", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1249-L1252", "path": "inference/model.py", "line": 1249, "end_line": 1252}, {"__type__": "CodeReference", "label": "inference/model.py L1262-L1263", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1262-L1263", "path": "inference/model.py", "line": 1262, "end_line": 1263}, {"__type__": "CodeReference", "label": "inference/config.json L43-L48", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L43-L48", "path": "inference/config.json", "line": 43, "end_line": 48}, {"__type__": "CodeReference", "label": "inference/config.json L63-L64", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L63-L64", "path": "inference/config.json", "line": 63, "end_line": 64}], "formula": "X'[i_{x}, i_{n}, i_{m}] = X[i_{x}, i_{n}, i_{m}] + g[i_{x}, i_{n}]\\, v[i_{x}, i_{m}]", "drawing": null}}, "1317094583": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1317094583, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "782676833": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 782676833, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "59633430": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 59633430, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1579951363": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1579951363, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2821525": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2821525, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "580393522": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 580393522, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1691174719": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1691174719, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1360358278": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1360358278, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "48548399": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 48548399, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "33631456": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 33631456, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "566270736": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 566270736, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "744347524": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 744347524, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "467090670": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 467090670, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1755693128": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1755693128, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1591096734": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1591096734, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1762160233": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1762160233, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "775507976": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 775507976, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "997586227": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 997586227, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "996088008": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 996088008, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "823757020": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 823757020, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2002171591": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2002171591, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2058971508": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2058971508, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2071905577": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2071905577, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "874077963": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 874077963, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1965436983": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1965436983, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "602528477": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 602528477, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "73897497": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 73897497, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "391693042": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 391693042, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1715833382": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1715833382, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "106528197": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 106528197, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "831279800": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 831279800, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1481352516": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1481352516, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "461033307": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 461033307, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2070745838": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2070745838, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "355812512": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 355812512, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "587693940": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 587693940, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "874217176": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 874217176, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "587172175": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 587172175, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1279002555": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1279002555, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "34495910": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 34495910, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "775188090": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 775188090, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "631582986": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 631582986, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1509541718": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1509541718, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "847706957": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 847706957, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1791112783": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1791112783, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1186080503": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1186080503, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "579128630": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 579128630, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "839195616": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 839195616, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1745119648": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1745119648, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "36149506": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 36149506, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2077229579": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2077229579, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "989792445": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 989792445, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "946923697": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 946923697, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "405840512": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 405840512, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "621171469": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 621171469, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1279846467": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1279846467, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "577147690": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 577147690, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1763031525": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1763031525, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "946624085": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 946624085, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "577388351": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 577388351, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1097899262": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1097899262, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "335544024": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 335544024, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1124303090": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1124303090, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1909332038": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1909332038, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1783401020": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1783401020, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "27430149": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 27430149, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "725001055": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 725001055, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1828261256": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1828261256, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "992914208": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 992914208, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1079899604": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1079899604, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2071926669": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2071926669, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "996737196": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 996737196, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1418720874": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1418720874, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "904930783": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 904930783, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2132621922": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2132621922, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "540928477": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 540928477, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "354609716": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 354609716, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1237800545": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1237800545, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1011221432": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1011221432, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "877875385": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 877875385, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "213642270": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 213642270, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "801975698": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 801975698, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Sliding Window Layer}", "description": "Layer 1, one of the two sliding-window layers: a sliding-window attention sublayer and a mixture sublayer, each inside the four-stream residual. The Engram of layer 1 has been added to the residual read by this layer.", "fill_color": "#EFEFF4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L26", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L26", "path": "inference/config.json", "line": 26, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L52", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L52", "path": "inference/config.json", "line": 52, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L1262-L1263", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1262-L1263", "path": "inference/model.py", "line": 1262, "end_line": 1263}], "formula": null, "drawing": null}}, "1565092097": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1565092097, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1720489762": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1720489762, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1261954404": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1261954404, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1216177087": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1216177087, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "797491979": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 797491979, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "650393550": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 650393550, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "177853509": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 177853509, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1359978287": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1359978287, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "224599997": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 224599997, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "327301158": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 327301158, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "233950338": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 233950338, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "212388305": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 212388305, "_name": {"__type__": "DynamicName", "body": "\\beta", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1008931453": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1008931453, "_name": {"__type__": "DynamicName", "body": "\\kappa", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "955513790": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 955513790, "_name": {"__type__": "DynamicName", "body": "\\mathrm{lo}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1257742509": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1257742509, "_name": {"__type__": "DynamicName", "body": "\\mathrm{hi}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "1932387484": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1932387484, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "461707635": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 461707635, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "874601708": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 874601708, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2072467948": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2072467948, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Rotary Embedding}", "description": "Each vector is rotated by the position of its token. The operation is called a rotary embedding. The last 64 of the 512 channels of each vector are taken two at a time as points in a plane. The rotation turns each point about the origin. The angle of the turn grows with the position of the token. Each pair of channels turns at a speed of its own. A query and a key that are both turned score against each other by how far apart their tokens are, because the product of two turned points depends on the difference of the two angles. The other 448 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels. The released code rotates the query of every head and the window latent of every token this way. The window latent is both the key and the value of its token.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L392-L406", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L406", "path": "inference/model.py", "line": 392, "end_line": 406}, {"__type__": "CodeReference", "label": "inference/model.py L771-L772", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L771-L772", "path": "inference/model.py", "line": 771, "end_line": 772}, {"__type__": "CodeReference", "label": "inference/model.py L705-L706", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L705-L706", "path": "inference/model.py", "line": 705, "end_line": 706}, {"__type__": "CodeReference", "label": "inference/model.py L368-L389", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L368-L389", "path": "inference/model.py", "line": 368, "end_line": 389}, {"__type__": "CodeReference", "label": "inference/model.py L680-L687", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L680-L687", "path": "inference/model.py", "line": 680, "end_line": 687}], "formula": "\\begin{gathered} y[i_{x}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{x}, |\\bar{z}| + 2 i_{t} + 1] = F[i_{x}, i_{t}]\\, \\big(v[i_{x}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{x}, |\\bar{z}| + 2 i_{t} + 1]\\big) \\\\ y[i_{x}, i_{\\bar{z}}] = v[i_{x}, i_{\\bar{z}}] \\end{gathered}", "drawing": null}}, "1362413828": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1362413828, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "891172938": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 891172938, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "434979618": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 434979618, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2035341104": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2035341104, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1949577967": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1949577967, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1185379382": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1185379382, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1535090370": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1535090370, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1198505538": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1198505538, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1654720364": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1654720364, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "70307082": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 70307082, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "317068880": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 317068880, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "249788935": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 249788935, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1857145049": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1857145049, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "415882848": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 415882848, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "918841960": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 918841960, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1152528281": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1152528281, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1428130650": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1428130650, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1229348667": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1229348667, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "1343977221": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1343977221, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "774326000": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 774326000, "_name": {"__type__": "DynamicName", "body": "b", "subscript": null, "settings": null, "code_form": "entries", "exponent": null}}, "_size": {"__ref__": 662338339}}, "679050082": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 679050082, "_name": {"__type__": "DynamicName", "body": "a", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "compression_ratio", "exponent": {"__type__": "DynamicName", "body": "2", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 2067311870}}, "1737197619": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1737197619, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{b}, i_{a}] = x[|a|_{2}\\, i_{b} + i_{a}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "576337652": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 576337652, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1064471311": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1064471311, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "661620168": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 661620168, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Compressor}", "description": "In the ratio-2 compressor two consecutive tokens form one entry. Each token is projected to a value and to a gate. A softmax over the two tokens of each channel weights the values. The weighted sum is normalised.", "fill_color": "#B8D8CE", "references": [{"__type__": "CodeReference", "label": "inference/model.py L429", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L429", "path": "inference/model.py", "line": 429, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L475", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L475", "path": "inference/model.py", "line": 475, "end_line": null}], "formula": null, "drawing": null}}, "550603394": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 550603394, "_name": {"__type__": "DynamicName", "body": "d", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "indexer_width_size", "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1161708895": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1161708895, "_name": {"__type__": "DynamicName", "body": "d", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "indexer_width", "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 550603394}}, "1615253697": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1615253697, "_name": {"__type__": "DynamicName", "body": "\\bar{d}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "unrotated_indexer_width", "exponent": {"__type__": "DynamicName", "body": "64", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__type__": "Addition", "content": [{"__ref__": 550603394}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -2}, {"__ref__": 1181195560}]}]}}, "599264257": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 599264257, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1587762189": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1587762189, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "521854120": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 521854120, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "999720132": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 999720132, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "528648230": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 528648230, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{d}}] = u[i_{\\bar{d}}], \\qquad y[|\\bar{d}| + i_{z}] = r[i_{z}], \\qquad |d| = |\\bar{d}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1466572087": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1466572087, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Rotary Embedding of the Encoder Indexer Keys}", "description": "Each indexer key of an encoder group is made from one compressed entry, which stands for two consecutive tokens. The key is rotated by the position of the first of the two tokens, which is twice the index of the entry. The last 64 of the 128 channels of each vector are taken two at a time as points in a plane. The rotation turns each point about the origin. The other 64 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L537-L545", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L537-L545", "path": "inference/model.py", "line": 537, "end_line": 545}, {"__type__": "CodeReference", "label": "inference/model.py L392-L406", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L406", "path": "inference/model.py", "line": 392, "end_line": 406}], "formula": "\\begin{gathered} y[i_{b}, |\\bar{d}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{b}, |\\bar{d}| + 2 i_{t} + 1] = F[i_{b}, i_{t}]\\, \\big(v[i_{b}, |\\bar{d}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{b}, |\\bar{d}| + 2 i_{t} + 1]\\big) \\\\ y[i_{b}, i_{\\bar{d}}] = v[i_{b}, i_{\\bar{d}}] \\end{gathered}", "drawing": null}}, "1592755125": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1592755125, "_name": {"__type__": "DynamicName", "body": "\\hat{E}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "indexer_scale_groups_size", "exponent": {"__type__": "DynamicName", "body": "4", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "2110537112": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 2110537112, "_name": {"__type__": "DynamicName", "body": "\\hat{E}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "indexer_scale_groups", "exponent": {"__type__": "DynamicName", "body": "4", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1592755125}}, "539533571": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 539533571, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\hat{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2101533416": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2101533416, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1855203620": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1855203620, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1361387321": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1361387321, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1613351041": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1613351041, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-126})", "description": "Keeps the scale of a group at or above 2^-126, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-126 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L165", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L165", "path": "inference/kernel.py", "line": 165, "end_line": null}], "formula": "y = \\max(x, 2^{-126})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "111961851": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 111961851, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "908261347": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 908261347, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2059861221": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2059861221, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "594278629": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 594278629, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "384884021": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 384884021, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1062077389": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1062077389, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}] = x[i_{\\hat{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1536568663": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1536568663, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "891518140": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 891518140, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{FP4 Round Trip in the Indexer}", "description": "The box rounds one indexer key or one indexer query the way the released indexer scores with it. The channels are cut into groups of 32. Each group is divided by a scale of its own. Every channel is rounded to the nearest number of the four-bit E2M1 form. The group is multiplied by its scale again. The E2M1 numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The scale of a group is its largest magnitude divided by 6 and rounded up to a power of two. The scale is therefore stored as an exponent alone, which is the E8M0 form. A logarithm to base two, a ceiling and a power of two state the rounding of the scale. The ceiling is a generic operator because the package has no numeric for it. The keys of every entry are kept for the whole context. The released report gives the smaller key cache and the faster scoring as the reasons for four bits. A system that keeps the keys as four-bit numbers with one scale per group holds them in about a quarter of the memory of 16-bit keys and can score them on the hardware path for four-bit numbers. The released code is a reference implementation and takes neither saving. It rounds a key and a query in place after it has rotated them, writes the rounded values back as 16-bit numbers, keeps the key cache in 16-bit, and scores with the ordinary 16-bit multiply. Its scores equal those of the four-bit system, because reading a four-bit key back gives the rounded number returned by this box. The rounding is therefore kept. The saving is not taken.", "fill_color": "#E6E1F4", "references": [{"__type__": "CodeReference", "label": "inference/model.py L546", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L546", "path": "inference/model.py", "line": 546, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L552", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L552", "path": "inference/model.py", "line": 552, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L27-L30", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L27-L30", "path": "inference/model.py", "line": 27, "end_line": 30}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L164-L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L164-L166", "path": "inference/kernel.py", "line": 164, "end_line": 166}, {"__type__": "CodeReference", "label": "inference/kernel.py L22-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L37", "path": "inference/kernel.py", "line": 22, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}], "formula": "\\begin{gathered} a[i_{\\hat{E}}] = \\max_{i_{\\hat{y}} \\in \\hat{y}} \\lvert v[\\lvert \\hat{y} \\rvert\\, i_{\\hat{E}} + i_{\\hat{y}}] \\rvert \\\\ s[i_{\\hat{E}}] = 2^{\\lceil \\log_{2} \\max(a[i_{\\hat{E}}] / 6,\\, 2^{-126}) \\rceil} \\\\ \\hat{v}[\\lvert \\hat{y} \\rvert\\, i_{\\hat{E}} + i_{\\hat{y}}] = s[i_{\\hat{E}}]\\, \\mathrm{E2M1}\\big(\\ulcorner v[\\lvert \\hat{y} \\rvert\\, i_{\\hat{E}} + i_{\\hat{y}}] / s[i_{\\hat{E}}] \\lrcorner_{-6}^{6}\\big) \\end{gathered}", "drawing": null}}, "1802186739": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1802186739, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "123359101": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 123359101, "_name": {"__type__": "DynamicName", "body": "i", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "indexer_heads_size", "exponent": {"__type__": "DynamicName", "body": "32", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1405909492": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1405909492, "_name": {"__type__": "DynamicName", "body": "i", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "indexer_heads", "exponent": {"__type__": "DynamicName", "body": "32", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 123359101}}, "2125466519": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2125466519, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2015865941": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2015865941, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1224550205": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1224550205, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "238171538": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 238171538, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "942789183": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 942789183, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{d}}] = u[i_{\\bar{d}}], \\qquad y[|\\bar{d}| + i_{z}] = r[i_{z}], \\qquad |d| = |\\bar{d}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1827969932": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1827969932, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Rotary Embedding of the Indexer Queries}", "description": "Each indexer query is rotated by the position of its token, once for each of the 32 indexer heads. The last 64 of the 128 channels of each vector are taken two at a time as points in a plane. The rotation turns each point about the origin. The angle of the turn grows with the position of the token. Each pair of channels turns at a speed of its own. A query and a key that are both turned score against each other by how far apart their tokens are, because the product of two turned points depends on the difference of the two angles. The other 64 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels. The indexer holds no table of its own. Its queries and keys are rotated with the table of the attention layer it belongs to.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L550-L551", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L550-L551", "path": "inference/model.py", "line": 550, "end_line": 551}, {"__type__": "CodeReference", "label": "inference/model.py L733-L734", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L733-L734", "path": "inference/model.py", "line": 733, "end_line": 734}, {"__type__": "CodeReference", "label": "inference/model.py L392-L406", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L406", "path": "inference/model.py", "line": 392, "end_line": 406}], "formula": "\\begin{gathered} y[i_{x}, |\\bar{d}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{x}, |\\bar{d}| + 2 i_{t} + 1] = F[i_{x}, i_{t}]\\, \\big(v[i_{x}, |\\bar{d}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{x}, |\\bar{d}| + 2 i_{t} + 1]\\big) \\\\ y[i_{x}, i_{\\bar{d}}] = v[i_{x}, i_{\\bar{d}}] \\end{gathered}", "drawing": null}}, "928635985": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 928635985, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\hat{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "992487066": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 992487066, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1812381864": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1812381864, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "350975716": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 350975716, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "499123753": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 499123753, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-126})", "description": "Keeps the scale of a group at or above 2^-126, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-126 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L165", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L165", "path": "inference/kernel.py", "line": 165, "end_line": null}], "formula": "y = \\max(x, 2^{-126})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1726735905": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1726735905, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "878682250": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 878682250, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1953023998": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1953023998, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1797140088": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1797140088, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1334351149": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1334351149, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "964101004": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 964101004, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}] = x[i_{\\hat{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "556932553": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 556932553, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2098311577": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 2098311577, "_name": {"__type__": "DynamicName", "body": "r|b", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 774326000}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__ref__": 662338339}}, "1408518067": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1408518067, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{b}, i_{r|b}] = x[i_{b} - i_{r|b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "907456827": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 907456827, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}, "shift": {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}]}, "extent": null}, "662862622": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 662862622, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. The input does not carry every axis read by the row, so one copy of the input is written for every position of the missing axes. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|a|_{2}\\, i_{b} + i_{a} - 1 + |a|_{2}] = x[i_{b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "176650506": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 176650506, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "(\\lvert d \\rvert \\lvert i \\rvert)^{-1/2} x", "description": "This map multiplies the weight of every indexer head by 1/64. The released indexer multiplies by one over the square root of the key width, which is 128, and by one over the square root of the head count, which is 32. The model drawn without the omitted mechanisms folds this factor into the weight `w^{I}`. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L513", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L513", "path": "inference/model.py", "line": 513, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L555", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L555", "path": "inference/model.py", "line": 555, "end_line": null}], "formula": "y = x (|d|_{128} |i|_{32})^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "140301595": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 140301595, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Indexer Scores}", "description": "One query is scored against every compressed entry reached by the query. The entries read by the attention are chosen from the scores. Each of the 32 indexer heads takes the product of its rotated and rounded indexer query with the key of the entry. A negative product is set to zero. The 32 results are summed under one weight per head, which is read off the hidden state of the token. The weights are multiplied by one over the square root of the key width times the head count, which is 1/64 in the released model. The released code computes the product with an ordinary 16-bit multiply over keys and queries that were rounded to four bits and written back as 16-bit numbers. The score is therefore the one given by a four-bit multiply. The reference implementation takes neither the smaller key cache nor the faster scoring.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L513", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L513", "path": "inference/model.py", "line": 513, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L515", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L515", "path": "inference/model.py", "line": 515, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L555-L557", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L555-L557", "path": "inference/model.py", "line": 555, "end_line": 557}], "formula": null, "drawing": null}}, "1646203631": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1646203631, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Lightning Indexer}", "description": "The lightning indexer scores every compressed entry reached by a query, so that the attention reads the best-scoring entries alone. The indexer query of every head is projected from the low-rank query latent, rotated by the position of its token and rounded to the four-bit FP4 format. The shared indexer keys arrive rotated and rounded. The keys are read back from every entry and merged onto the queries, so each query holds the keys of the entries reached by that query. The scoring box scores one query against those keys. The rounding makes the scores equal those of a system that keeps the keys as four-bit numbers and scores them on the four-bit hardware path. The released code is a reference implementation. It keeps the rounded keys and queries as 16-bit numbers and scores in 16-bit, so it takes neither the smaller cache nor the faster scoring.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L550-L552", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L550-L552", "path": "inference/model.py", "line": 550, "end_line": 552}, {"__type__": "CodeReference", "label": "inference/model.py L563-L565", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L563-L565", "path": "inference/model.py", "line": 563, "end_line": 565}, {"__type__": "CodeReference", "label": "inference/model.py L567", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L567", "path": "inference/model.py", "line": 567, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L577-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L577-L580", "path": "inference/model.py", "line": 577, "end_line": 580}], "formula": null, "drawing": null}}, "454012108": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 454012108, "_name": {"__type__": "DynamicName", "body": "s", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "512", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1794391610": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1794391610, "_name": {"__type__": "DynamicName", "body": "s|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "512", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 454012108}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}, "shift": {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}]}, "extent": null}, "1681018557": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1681018557, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the positions of the largest scores along the axis consumed and hands out no score, because nothing reads a score after the selection. Each result is a natural number, one per slot of the result's axis. A slot with no entry to hold reads the unit.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L579", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L579", "path": "inference/model.py", "line": 578, "end_line": 579}, {"__type__": "CodeReference", "label": "inference/model.py L607", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L607", "path": "inference/model.py", "line": 607, "end_line": null}], "formula": "\\{p[i_{s|x}]\\} = \\{i_{r|x} : s[i_{r|x}] \\text{ is among the } |s|_{512} \\text{ largest of } s\\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "318562117": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 318562117, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in INT64 and writes it in INT32. The written format holds an element in 32 bits against the 64 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{INT64} \\to \\mathtt{INT32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1031323421": {"__type__": "TapeSlot", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "TapeSlot"}, "_id": 1031323421, "_name": {"__type__": "DynamicName", "body": "\\mathrm{sel}", "subscript": {"__type__": "DynamicName", "body": "b", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": null, "code_form": null, "exponent": null}}}, "1000362549": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1000362549, "_name": {"__type__": "DynamicName", "body": "l", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "625578458": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 625578458, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1919548415": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1919548415, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2038095376": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2038095376, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "850681712": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 850681712, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "629961344": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 629961344, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2038245482": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2038245482, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Rotary Embedding of the Encoder Entries}", "description": "Each compressed entry of an encoder group stands for two consecutive tokens. The entry is rotated by the position of the first of the two, which is twice the index of the entry. The last 64 of the 512 channels of each vector are taken two at a time as points in a plane. The rotation turns each point about the origin. The other 448 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels. The indexer keys are made from the entries before the rotation. The attention of this layer and of every layer that reuses the entries reads the entries after the rotation.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L749-L758", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L749-L758", "path": "inference/model.py", "line": 749, "end_line": 758}, {"__type__": "CodeReference", "label": "inference/model.py L392-L406", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L406", "path": "inference/model.py", "line": 392, "end_line": 406}, {"__type__": "CodeReference", "label": "inference/model.py L85-L87", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L85-L87", "path": "inference/model.py", "line": 85, "end_line": 87}], "formula": "\\begin{gathered} y[i_{b}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{b}, |\\bar{z}| + 2 i_{t} + 1] = F[i_{b}, i_{t}]\\, \\big(v[i_{b}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{b}, |\\bar{z}| + 2 i_{t} + 1]\\big) \\\\ y[i_{b}, i_{\\bar{z}}] = v[i_{b}, i_{\\bar{z}}] \\end{gathered}", "drawing": null}}, "1264779141": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1264779141, "_name": {"__type__": "DynamicName", "body": "E", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "32", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "365519784": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 365519784, "_name": {"__type__": "DynamicName", "body": "E", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "32", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1264779141}}, "1884663983": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1884663983, "_name": {"__type__": "DynamicName", "body": "y", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "channels_per_entry_scale_size", "exponent": {"__type__": "DynamicName", "body": "16", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1889059728": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1889059728, "_name": {"__type__": "DynamicName", "body": "y", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "channels_per_entry_scale", "exponent": {"__type__": "DynamicName", "body": "16", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1884663983}}, "619966234": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 619966234, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{E}, i_{y}] = x[|y|_{16}\\, i_{E} + i_{y}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "710177950": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 710177950, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "722942182": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 722942182, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1069235466": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1069235466, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "252971105": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 252971105, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-9})", "description": "Keeps the scale of a group at or above 2^-9, the smallest number above zero held by the E4M3 form, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-9 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L161-L162", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L161-L162", "path": "inference/kernel.py", "line": 161, "end_line": 162}], "formula": "y = \\max(x, 2^{-9})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1005850860": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1005850860, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2101239643": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2101239643, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "21926533": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 21926533, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "739293908": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 739293908, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1210695560": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1210695560, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "214680912": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 214680912, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|y|_{16}\\, i_{E} + i_{y}] = x[i_{E}, i_{y}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1618522984": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1618522984, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1078094134": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1078094134, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{FP4 Round Trip of an Entry}", "description": "The box rounds one compressed entry the way the released cache holds it. The channels are cut into groups of 16. Each group is divided by a scale of its own. Every channel is rounded to the nearest number of the four-bit E2M1 form. The group is multiplied by its scale again. The E2M1 numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The scale of a group is its largest magnitude divided by 6, so the largest channel of the group divides to 6. The scale is itself rounded to eight bits in the E4M3 form. The entries are kept for the whole context. They are therefore the largest thing held in memory by a request while text is generated. Four bits for each channel take close to half the memory of the eight bits used by DeepSeek-V4. The released code rounds an entry in place after it has rotated the entry, so every layer that reads the cache reads the rounded values.", "fill_color": "#E6E1F4", "references": [{"__type__": "CodeReference", "label": "inference/model.py L758-L761", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L758-L761", "path": "inference/model.py", "line": 758, "end_line": 761}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L160-L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L160-L163", "path": "inference/kernel.py", "line": 160, "end_line": 163}, {"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}], "formula": "\\begin{gathered} a[i_{E}] = \\max_{i_{y} \\in y} \\lvert v[\\lvert y \\rvert\\, i_{E} + i_{y}] \\rvert \\\\ s[i_{E}] = \\mathrm{E4M3}\\big(\\max(a[i_{E}] / 6,\\, 2^{-9})\\big) \\\\ \\hat{v}[\\lvert y \\rvert\\, i_{E} + i_{y}] = s[i_{E}]\\, \\mathrm{E2M1}\\big(\\ulcorner v[\\lvert y \\rvert\\, i_{E} + i_{y}] / s[i_{E}] \\lrcorner_{-6}^{6}\\big) \\end{gathered}", "drawing": null}}, "907787552": {"__type__": "TapeSlot", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "TapeSlot"}, "_id": 907787552, "_name": {"__type__": "DynamicName", "body": "\\mathrm{ckv}", "subscript": {"__type__": "DynamicName", "body": "b", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": null, "code_form": null, "exponent": null}}}, "1916780111": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1916780111, "_name": {"__type__": "DynamicName", "body": "r|b", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 774326000}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__ref__": 662338339}}, "1901887635": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1901887635, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{b}, i_{r|b}] = x[i_{b} - i_{r|b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "952260271": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 952260271, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}, "shift": {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}]}, "extent": null}, "1164531166": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1164531166, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. The input does not carry every axis read by the row, so one copy of the input is written for every position of the missing axes. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|a|_{2}\\, i_{b} + i_{a} - 1 + |a|_{2}] = x[i_{b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "740094440": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 740094440, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1521600677": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1521600677, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Entry Gather}", "description": "The gather reads the latents of the selected entries. The compressed entries are read back from every entry and merged onto the queries the way the indexer reads its keys. One latent is read per slot at the distance held by the selection for that slot.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": null, "drawing": null}}, "803747460": {"__type__": "ConcatenatedAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "ConcatenatedAxis"}, "_id": 803747460, "_name": null}, "_size": {"__type__": "Addition", "content": [{"__ref__": 454012108}, {"__ref__": 990851622}]}, "parts": [{"__ref__": 1229348667}, {"__ref__": 1794391610}]}, "1182401965": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1182401965, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation}", "description": "Writes each input onto its own run of the concatenated axis. The second run comes after the first. The size of that axis is therefore the sum of the sizes of the parts. Every other axis is carried through unchanged.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L778", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L778", "path": "inference/model.py", "line": 778, "end_line": null}], "formula": "y = x^{(1)} \\,\\Vert\\, x^{(2)}, \\quad |t| = |w| + |s|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1915689096": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1915689096, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1067861255": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1067861255, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "306592261": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 306592261, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "375590766": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 375590766, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "383691764": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 383691764, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots, the selected entries and the learned sink. The two kinds of latent are laid end to end along one slot axis. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "161506428": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 161506428, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1377146642": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1377146642, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1741076476": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1741076476, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1628924864": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1628924864, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1120210550": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1120210550, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1992358532": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1992358532, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Inverse Rotary Embedding}", "description": "The last 64 of the 512 channels of each attention output are taken two at a time as points in a plane. Each point is turned back by the angle of the position of its query. The output of a query is a weighted sum of latents. Each latent in the sum was turned by the angle of its own position. That position is the token for a window latent or a decoder entry, and the first token of its group for an encoder entry. Each latent in the sum is then left turned by the difference between that position and the position of the query, so the output carries no position counted from the start of the context. The other 448 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels. The map after the circle takes the conjugate of every turn, which is the turn through the same angle in the other direction.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L780-L781", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780-L781", "path": "inference/model.py", "line": 780, "end_line": 781}, {"__type__": "CodeReference", "label": "inference/model.py L392-L395", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L395", "path": "inference/model.py", "line": 392, "end_line": 395}, {"__type__": "CodeReference", "label": "inference/model.py L398-L399", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L398-L399", "path": "inference/model.py", "line": 398, "end_line": 399}], "formula": "\\begin{gathered} y[i_{x}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{x}, |\\bar{z}| + 2 i_{t} + 1] = F[i_{x}, i_{t}]\\, \\big(v[i_{x}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{x}, |\\bar{z}| + 2 i_{t} + 1]\\big) \\\\ y[i_{x}, i_{\\bar{z}}] = v[i_{x}, i_{\\bar{z}}] \\end{gathered}", "drawing": null}}, "65270232": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 65270232, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "757401061": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 757401061, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "205644247": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 205644247, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Full Attention}", "description": "The Full layer of an encoder group compresses every two consecutive tokens of its hidden state into one entry and makes one indexer key from each entry. The layer rotates and rounds both as the released caches hold them. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. The layer scores every entry reached by each query and keeps the 512 best. The layer then attends over the window and the kept entries under one softmax. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection. The entries and the selection are stored on the tape, which holds what one layer leaves for another. The Reuse layers of the same group read the entries and the selection there.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L739-L763", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L739-L763", "path": "inference/model.py", "line": 739, "end_line": 763}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}, {"__type__": "CodeReference", "label": "inference/model.py L1166-L1176", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1176", "path": "inference/model.py", "line": 1166, "end_line": 1176}], "formula": null, "drawing": null}}, "1798776363": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1798776363, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "558565537": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 558565537, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2005853479": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2005853479, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "141017435": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 141017435, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1474027241": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1474027241, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "809174183": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 809174183, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1713686251": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1713686251, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "576429634": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 576429634, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1551474592": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1551474592, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2093019819": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2093019819, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1035646472": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1035646472, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1100875623": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1100875623, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "579801691": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 579801691, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1186399858": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1186399858, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "851089758": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 851089758, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1977753541": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1977753541, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1497531933": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1497531933, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1763523738": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1763523738, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "278164091": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 278164091, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "209624389": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 209624389, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "9721471": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 9721471, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "557291023": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 557291023, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1806452263": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1806452263, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "908097070": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 908097070, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "677005823": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 677005823, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1632473942": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1632473942, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2085752804": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2085752804, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "983786086": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 983786086, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2029219969": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2029219969, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "299554026": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 299554026, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1884332483": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1884332483, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1126543339": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1126543339, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "624322775": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 624322775, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "802264865": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 802264865, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "646698256": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 646698256, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "350916789": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 350916789, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "169511678": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 169511678, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "140208737": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 140208737, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "2090768131": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2090768131, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "600612723": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 600612723, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2049491507": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2049491507, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "210402297": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 210402297, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "54652736": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 54652736, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1601629184": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1601629184, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1812400898": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1812400898, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1672795870": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1672795870, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1127661722": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1127661722, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "264670751": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 264670751, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1218783882": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1218783882, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "784698223": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 784698223, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "934405402": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 934405402, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "35994271": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 35994271, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1131662858": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1131662858, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "159665161": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 159665161, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1344620530": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1344620530, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1516840348": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1516840348, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "44366383": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 44366383, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1193818320": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1193818320, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1790359033": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1790359033, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1964077878": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1964077878, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "679263556": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 679263556, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "774604280": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 774604280, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1577530757": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1577530757, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1923243870": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1923243870, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1762171389": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1762171389, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1569498193": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1569498193, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "655091204": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 655091204, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1028312327": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1028312327, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1583162821": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1583162821, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "658671640": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 658671640, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "470356694": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 470356694, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2139267218": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 2139267218, "_name": {"__type__": "DynamicName", "body": "r|b", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 774326000}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__ref__": 662338339}}, "1485512374": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1485512374, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{b}, i_{r|b}] = x[i_{b} - i_{r|b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1137467301": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1137467301, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}, "shift": {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}]}, "extent": null}, "1214802731": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1214802731, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. The input does not carry every axis read by the row, so one copy of the input is written for every position of the missing axes. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|a|_{2}\\, i_{b} + i_{a} - 1 + |a|_{2}] = x[i_{b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "483322766": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 483322766, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2028932702": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2028932702, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Entry Gather}", "description": "The gather reads the latents of the selected entries. The compressed entries are read back from every entry and merged onto the queries the way the indexer reads its keys. One latent is read per slot at the distance held by the selection for that slot.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": null, "drawing": null}}, "1847314369": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1847314369, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation}", "description": "Writes each input onto its own run of the concatenated axis. The second run comes after the first. The size of that axis is therefore the sum of the sizes of the parts. Every other axis is carried through unchanged.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L778", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L778", "path": "inference/model.py", "line": 778, "end_line": null}], "formula": "y = x^{(1)} \\,\\Vert\\, x^{(2)}, \\quad |t| = |w| + |s|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1263837627": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1263837627, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "947903668": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 947903668, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "344548427": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 344548427, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "588456156": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 588456156, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1070020896": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1070020896, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots, the selected entries and the learned sink. The two kinds of latent are laid end to end along one slot axis. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "485997434": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 485997434, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2128047686": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2128047686, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "329160556": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 329160556, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1688239023": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1688239023, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "877051688": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 877051688, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1210411728": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1210411728, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "497897701": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 497897701, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "56335649": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 56335649, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reuse Attention}", "description": "A Reuse layer reads the entries and the selection from the tape, which holds what an earlier layer left for it. The layer therefore compresses no tokens and scores no entries. The entries arrive rotated and rounded. The layer computes its own query and window latents from its own hidden state. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. The layer reads the latents of the selected entries and attends over the window and those entries under one softmax. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection.", "fill_color": "#B8D8CE", "references": [{"__type__": "CodeReference", "label": "inference/model.py L1166-L1176", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1176", "path": "inference/model.py", "line": 1166, "end_line": 1176}, {"__type__": "CodeReference", "label": "inference/model.py L1180", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1180", "path": "inference/model.py", "line": 1180, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L762-L763", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L762-L763", "path": "inference/model.py", "line": 762, "end_line": 763}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}], "formula": null, "drawing": null}}, "1249588417": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1249588417, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2067117292": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2067117292, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1059963016": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1059963016, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "943457697": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 943457697, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "573961373": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 573961373, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1156629493": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1156629493, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "437052142": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 437052142, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1034562481": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1034562481, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "142184949": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 142184949, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "3582023": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 3582023, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1262990274": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1262990274, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1252589602": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1252589602, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "414518624": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 414518624, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "848803571": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 848803571, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1190542461": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1190542461, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2032366042": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2032366042, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "401749686": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 401749686, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "537679409": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 537679409, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "416265067": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 416265067, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2010668387": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2010668387, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "359932494": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 359932494, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1978505263": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1978505263, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2144075072": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2144075072, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "812238814": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 812238814, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1243256156": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1243256156, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "576638543": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 576638543, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "600491194": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 600491194, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1578633064": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1578633064, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "675017027": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 675017027, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1009859751": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1009859751, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "734703125": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 734703125, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1970866751": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1970866751, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "108826540": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 108826540, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "395259675": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 395259675, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "603781768": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 603781768, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "461172494": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 461172494, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "440942973": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 440942973, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1760825430": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1760825430, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "1123630233": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1123630233, "_name": null}, "repetition": {"__type__": "Integer", "_value": 5}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reuse Layer}", "description": "One of the five Reuse layers of an encoder group: a Reuse attention sublayer reading the entries and the selection written by the group's Full layer, and a mixture sublayer, each inside the four-stream residual.", "fill_color": "#F3F3F4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L27-L28", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L27-L28", "path": "inference/config.json", "line": 27, "end_line": 28}, {"__type__": "CodeReference", "label": "inference/model.py L1166-L1170", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1170", "path": "inference/model.py", "line": 1166, "end_line": 1170}], "formula": null, "drawing": null}}, "423689127": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 423689127, "_name": {"__type__": "DynamicName", "body": "l", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "repetition": {"__type__": "Integer", "_value": 2}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Encoder Group}", "description": "One of the first two encoder groups, which are layers 2 to 13: a Full layer that compresses two tokens into each entry and writes the entries and the selection, followed by five Reuse layers that read them. The released model has three encoder groups. The third is drawn on its own after this block, because Engram is applied to the residual before its first layer.", "fill_color": "#EFEFF4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L27-L28", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L27-L28", "path": "inference/config.json", "line": 27, "end_line": 28}, {"__type__": "CodeReference", "label": "inference/config.json L52", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L52", "path": "inference/config.json", "line": 52, "end_line": null}, {"__type__": "CodeReference", "label": "config.json L106", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/config.json#L106", "path": "config.json", "line": 106, "end_line": null}, {"__type__": "CodeReference", "label": "config.json L112", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/config.json#L112", "path": "config.json", "line": 112, "end_line": null}], "formula": null, "drawing": null}}, "1947078872": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1947078872, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "957724017": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 957724017, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Token Map}", "description": "Replaces a token identifier by the identifier of the token's normalised text, so that tokens such as The, the and THE hash alike. The table M is fixed. The released code builds it once from the tokenizer. Training never changes it. The result of the table is a natural number below 99,092.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L17-L61", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L17-L61", "path": "inference/engram.py", "line": 17, "end_line": 61}, {"__type__": "CodeReference", "label": "inference/engram.py L164", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L164", "path": "inference/engram.py", "line": 164, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L48", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L48", "path": "inference/config.json", "line": 48, "end_line": null}], "formula": "\\mathrm{cmp}(\\mathrm{id})[i_{x}] = M[\\mathrm{id}[i_{x}]]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "587508430": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 587508430, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Token Map}", "description": "Each token identifier is replaced by the identifier of its normalised text. The normalisation lowercases the text, removes accents and joins runs of white space. The tokens The, the and THE therefore receive one identifier. The n-grams that hold them address the same rows. The map is a lookup in a table M that has one entry for every token identifier and returns one of 99,092 compressed identifiers. The table is fixed. The released code builds it once from the tokenizer. Training never changes it.", "fill_color": "#FCE0E1", "references": [{"__type__": "CodeReference", "label": "inference/engram.py L17-L61", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L17-L61", "path": "inference/engram.py", "line": 17, "end_line": 61}, {"__type__": "CodeReference", "label": "inference/engram.py L164", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L164", "path": "inference/engram.py", "line": 164, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L48", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L48", "path": "inference/config.json", "line": 48, "end_line": null}], "formula": "\\mathrm{cmp}(\\mathrm{id})[i_{x}] = M[\\mathrm{id}[i_{x}]]", "drawing": null}}, "949792920": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 949792920, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "ngram", "description": "The view is the lookback of Engram. Slot i_L of token i_x reads the token i_L places before it. Slot 0 is therefore the token itself. The four slots hold the longest n-gram. A slot that reaches before the first token holds no value. The guarded axis L|x records which slots hold no value. The same view reads the compressed identifiers and the modality.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L171-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L171-L172", "path": "inference/engram.py", "line": 171, "end_line": 172}], "formula": "y[i_{x}, i_{L|x}] = x[i_{x} - i_{L|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1192316171": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1192316171, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "p_{14}", "description": "The primes of the hash of layer 14 are one for each of the |G||K| pairs of an n-gram order and a hash head, in the order the released layout draws them: the successive primes above the row count given by the configuration for one pair, sixteen million. The datatype of the array is the naturals below the bound p-bar, which is one bound standing over every prime of the layer, because a wire carries one datatype. A remainder after division by a prime is below that prime, so the same bound holds every remainder as well. The table of layer 14 has 384,016,682 rows, which is the sum of its primes. The offset of a pair is the sum of the primes before it. The row ranges of the pairs are therefore disjoint.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L102-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L102-L118", "path": "inference/engram.py", "line": 102, "end_line": 118}], "formula": "p_{14}[0] < p_{14}[1] < \\dots < p_{14}[|G||K| - 1] < \\bar{p}, \\quad p_{14}[i_{GK}] \\text{ prime}, \\quad T_{14} = \\sum_{i_{GK} \\in GK} p_{14}[i_{GK}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "179209777": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 179209777, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\alpha_{14}", "description": "The multipliers of the hash of layer 14 are one odd integer per lookback slot. They are drawn once for that layer from a random number generator seeded by the layer, so the two Engram layers hash differently. An identifier is below the identifier bound. A multiplier is below 2^63 divided by that bound. The product of the two is therefore below 2^63 and is a 64-bit integer with nothing cast.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L64-L83", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L64-L83", "path": "inference/engram.py", "line": 64, "end_line": 83}], "formula": "\\alpha_{14}[i_{L}] < 2^{63} / \\hat{v}, \\quad \\alpha_{14}[i_{L}] \\text{ odd}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1276813122": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1276813122, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "prefix", "description": "The view is the prefix of the lookback slots read by each n-gram order. The order i_G reads the slot i_L' + i_G + 1 - |G| of the products. The order therefore covers the newest i_G + 2 slots. The slots not reached by the order hold no value. The exclusive or that follows ignores those slots, so one reduction gives the hash of the 2-gram, the 3-gram and the 4-gram.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L179-L184", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L179-L184", "path": "inference/engram.py", "line": 179, "end_line": 184}], "formula": "y[i_{G}, i_{L'|G}] = x[i_{G} + i_{L'|G} + 1 - |G|_{3}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2083782711": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2083782711, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pair", "description": "The view reads an array over the |G||K| pairs as an array over the n-gram orders and the hash heads. The pair |K| i_G + i_K stands at the order i_G and the head i_K. The released layout draws the primes in that order. The primes and the offsets are both read this way.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L108-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L108-L118", "path": "inference/engram.py", "line": 108, "end_line": 118}], "formula": "y[i_{G}, i_{K}] = x[|K|_{8}\\, i_{G} + i_{K}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "697474948": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 697474948, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "before", "description": "The view reads the primes that stand before each pair of an n-gram order and a hash head. The pair i_GK reads the prime at i_GK' + i_GK - |GK|. The primes before the pair therefore fill the last i_GK slots. The earlier slots hold no value. The sum over the slots is the offset of the pair. The rows of that pair begin at the offset in the table.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L108-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L108-L118", "path": "inference/engram.py", "line": 108, "end_line": 118}], "formula": "y[i_{GK}, i_{GK'|GK}] = x[i_{GK} + i_{GK'|GK} - |G|_{3} |K|_{8}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1895723426": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1895723426, "_name": {"__type__": "DynamicName", "body": "T", "subscript": {"__type__": "DynamicName", "body": "14", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "384016682", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "275602308": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 275602308, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pair", "description": "The view reads an array over the |G||K| pairs as an array over the n-gram orders and the hash heads. The pair |K| i_G + i_K stands at the order i_G and the head i_K. The released layout draws the primes in that order. The primes and the offsets are both read this way.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L108-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L108-L118", "path": "inference/engram.py", "line": 108, "end_line": 118}], "formula": "y[i_{G}, i_{K}] = x[|K|_{8}\\, i_{G} + i_{K}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1080949139": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1080949139, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\mathrm{cast}", "description": "Pyncd represents Natural values as a bounded datatype. Here the cast names the correct size for the subsequent embedding, which uses the bound to determine its size. The value read and the value written are both held in INT64.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/engram.py L184", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L184", "path": "inference/engram.py", "line": 184, "end_line": null}], "formula": "y = x, \\quad \\mathrm{Nat}(T_14:384016682 + \\bar{p}) \\to \\mathrm{Nat}(T_14:384016682)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1537651249": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1537651249, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{N-gram Hash}", "description": "An n-gram here is the run of the last two, three or four compressed identifiers that ends at a token. A table with one row for every possible n-gram would be too large, because 99,092 identifiers give about 10^20 different runs of four. Each n-gram is therefore hashed into one of about sixteen million rows per order and head. The hash is written out over integer operators. Each of the |L| identifiers is multiplied by the fixed odd multiplier of its slot. A multiplier is below 2^63 divided by the identifier bound, so every product is below 2^63 and is a 64-bit integer. The view named prefix reads, for each order, the products of the newest two, three or four slots. A slot not reached by the order holds the unit. The bitwise exclusive or of those products is one 64-bit integer per order. The exclusive or is defined on bit patterns, so the products carry a bound that is a power of two. The remainder of that integer after division by the prime of the order and the head is below that prime. The primes are one fixed array over the |G||K| pairs of an order and a head. The array holds the successive primes above 16,000,000 in the order the released layout draws them. The offset of a pair is the sum of the primes before it. The primes before a pair are read through the view named before and summed. The remainder plus the offset is therefore a row number in a range used by no other pair. The table has as many rows as the primes sum to. The primes and the offsets are then read over the orders and the heads. Two n-grams may share a row under one head and seldom share one under another, because the eight heads divide by eight different primes. The multipliers, the primes and the offsets belong to layer 14. The table addressed by that layer holds 384,016,682 rows, which is the sum of its 24 primes.", "fill_color": "#F5E6D9", "references": [{"__type__": "CodeReference", "label": "inference/engram.py L169-L175", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L169-L175", "path": "inference/engram.py", "line": 169, "end_line": 175}, {"__type__": "CodeReference", "label": "inference/engram.py L179-L184", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L179-L184", "path": "inference/engram.py", "line": 179, "end_line": 184}, {"__type__": "CodeReference", "label": "inference/engram.py L64-L83", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L64-L83", "path": "inference/engram.py", "line": 64, "end_line": 83}, {"__type__": "CodeReference", "label": "inference/engram.py L108-L118", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L108-L118", "path": "inference/engram.py", "line": 108, "end_line": 118}, {"__type__": "CodeReference", "label": "inference/engram.py L148-L149", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/engram.py#L148-L149", "path": "inference/engram.py", "line": 148, "end_line": 149}], "formula": "\\mathrm{hash}_{14}(c)[i_{G}, i_{K}] = \\Big( \\bigoplus_{i_{L} \\in L,\\; i_{L} \\leq i_{G} + 1} \\alpha_{14}[i_{L}]\\, c[i_{L}] \\Big) \\bmod p_{14}[i_{G}, i_{K}] + o_{14}[i_{G}, i_{K}]", "drawing": null}}, "494767011": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 494767011, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{N-gram Table}", "description": "Returns the learned row of width 256 named by a row number, from the n-gram table of layer 14, which holds 384,016,682 rows. One token looks up 24 rows, one for every n-gram order and hash head. The released code stores the rows in FP8 with one scale for every 32 channels and converts a row to reals when the row is read.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L296-L325", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L296-L325", "path": "inference/model.py", "line": 296, "end_line": 325}, {"__type__": "CodeReference", "label": "inference/config.json L45", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L45", "path": "inference/config.json", "line": 45, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L64", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L64", "path": "inference/config.json", "line": 64, "end_line": null}], "formula": "E_{14}(r)[i_{D}] = N_{14}[r, i_{D}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "469839001": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 469839001, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{N-gram Table}", "description": "Each of the 24 row numbers of a token, one for every n-gram order and hash head, is looked up in the learned table of this layer, which gives 24 vectors of width |D| = 256. One linear map reads the 24 vectors together and gives one key for each residual stream. A second map gives one value shared by every stream. The released code holds the two maps as one weight and cuts its result in two. The table of layer 14 has 384,016,682 rows, which is the sum of the 24 primes of the layer. The released code stores the rows in FP8 with one scale for every 32 channels and converts a row to reals when the row is read.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L309-L310", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L309-L310", "path": "inference/model.py", "line": 309, "end_line": 310}, {"__type__": "CodeReference", "label": "inference/model.py L317-L321", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L317-L321", "path": "inference/model.py", "line": 317, "end_line": 321}, {"__type__": "CodeReference", "label": "inference/model.py L343-L345", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L343-L345", "path": "inference/model.py", "line": 343, "end_line": 345}, {"__type__": "CodeReference", "label": "inference/model.py L353-L355", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L353-L355", "path": "inference/model.py", "line": 353, "end_line": 355}, {"__type__": "CodeReference", "label": "inference/config.json L45", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L45", "path": "inference/config.json", "line": 45, "end_line": null}], "formula": null, "drawing": null}}, "1686088331": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1686088331, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "815506369": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 815506369, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1256829401": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1256829401, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\mathrm{gate}", "description": "This map is the gate of the Engram module. The normalised dot product of a stream with its key is divided by the square root of the width. The magnitude of that product is kept at or above one millionth. The square root of the magnitude is taken. The sign of the dot product is put back. A sigmoid follows. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L341", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L341", "path": "inference/model.py", "line": 341, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L360-L362", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L360-L362", "path": "inference/model.py", "line": 360, "end_line": 362}], "formula": "y = \\sigma(\\operatorname{sign}(x |m|_{5120}^{-2^{-1}}) \\sqrt{\\max(\\lvert x |m|_{5120}^{-2^{-1}} \\rvert, \\varepsilon_{\\mathrm{g}})})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1348850376": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1348850376, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Engram Gate}", "description": "A stream receives the looked-up value multiplied by its gate. Each stream of a token is multiplied channel by channel with its key and with a learned weight. The products are summed over the channels. The sum is divided by the root mean square of the stream, by the root mean square of the key and by the square root of the width. Multiplying the stream or the key by a positive constant therefore leaves the result unchanged. The two root mean squares have an epsilon under the root and no learned gain. The gate is the sigmoid of the signed square root of that number. The gate is above one half where the sum is positive and below one half where the sum is negative. The released code multiplies the gate by one minus the modality of the token. The multiplication sets the gate to zero at an image token. A model that reads text alone holds no image token.", "fill_color": "#F7E0EF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L347-L348", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L347-L348", "path": "inference/model.py", "line": 347, "end_line": 348}, {"__type__": "CodeReference", "label": "inference/model.py L356-L362", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L356-L362", "path": "inference/model.py", "line": 356, "end_line": 362}, {"__type__": "CodeReference", "label": "inference/model.py L363-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L363-L364", "path": "inference/model.py", "line": 363, "end_line": 364}], "formula": "y[i_{x}, i_{n}] = \\frac{\\sum_{i_{m} \\in m} X[i_{x}, i_{n}, i_{m}]\\, w[i_{n}, i_{m}]\\, k[i_{x}, i_{n}, i_{m}]}{\\mathrm{rms}(X[i_{x}, i_{n}])\\, \\mathrm{rms}(k[i_{x}, i_{n}])}, \\quad g[i_{x}, i_{n}] = \\sigma\\Big( \\mathrm{sign}(y) \\sqrt{\\max(\\lvert y \\rvert \\lvert m \\rvert^{-1/2}, \\varepsilon_{\\mathrm{g}})} \\Big)", "drawing": null}}, "1153616329": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1153616329, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1626727668": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1626727668, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1664685496": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1664685496, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Engram of Layer 14}", "description": "Engram is a memory. The model reads the memory by looking up short runs of text. An n-gram here is the run of the last two, three or four token identifiers that ends at a token. The three n-grams of each token are hashed into row numbers of a large learned table. The rows are read. The rows are projected into one key for each of the four residual streams and one value shared by the streams. Each stream is compared with its key. The comparison gives a gate g between zero and one. The value v times the gate is added to the stream. The module reads the token identifiers from the tape slot written by the input of the model. The module runs on the residual that enters layer 14. Layers 1 and 14 each hold a module with a table and weights of its own.", "fill_color": "#E4EFD9", "references": [{"__type__": "CodeReference", "label": "inference/model.py L328-L365", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L328-L365", "path": "inference/model.py", "line": 328, "end_line": 365}, {"__type__": "CodeReference", "label": "inference/model.py L1249-L1252", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1249-L1252", "path": "inference/model.py", "line": 1249, "end_line": 1252}, {"__type__": "CodeReference", "label": "inference/model.py L1262-L1263", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1262-L1263", "path": "inference/model.py", "line": 1262, "end_line": 1263}, {"__type__": "CodeReference", "label": "inference/config.json L43-L48", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L43-L48", "path": "inference/config.json", "line": 43, "end_line": 48}, {"__type__": "CodeReference", "label": "inference/config.json L63-L64", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L63-L64", "path": "inference/config.json", "line": 63, "end_line": 64}], "formula": "X'[i_{x}, i_{n}, i_{m}] = X[i_{x}, i_{n}, i_{m}] + g[i_{x}, i_{n}]\\, v[i_{x}, i_{m}]", "drawing": null}}, "1932645023": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1932645023, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1461781655": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1461781655, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1926537084": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1926537084, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "544283397": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 544283397, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1011839818": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1011839818, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1294256242": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1294256242, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "562970999": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 562970999, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "424369608": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 424369608, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1401135142": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1401135142, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "933914619": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 933914619, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1921048670": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1921048670, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "310739725": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 310739725, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "726160625": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 726160625, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "442268571": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 442268571, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1395137796": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1395137796, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1862954308": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1862954308, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "670083067": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 670083067, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "49284138": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 49284138, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "787758973": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 787758973, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2106401017": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2106401017, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1730423003": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1730423003, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1309440657": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1309440657, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2080462902": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2080462902, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1979844634": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1979844634, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "212578221": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 212578221, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2060795254": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2060795254, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "620336985": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 620336985, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "818596443": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 818596443, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1480869452": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1480869452, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "380755848": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 380755848, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "215972085": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 215972085, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2107708861": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 2107708861, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "888560449": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 888560449, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "785548099": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 785548099, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{b}, i_{a}] = x[|a|_{2}\\, i_{b} + i_{a}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1498751400": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1498751400, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1713464253": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1713464253, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2092775892": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2092775892, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Compressor}", "description": "In the ratio-2 compressor two consecutive tokens form one entry. Each token is projected to a value and to a gate. A softmax over the two tokens of each channel weights the values. The weighted sum is normalised.", "fill_color": "#B8D8CE", "references": [{"__type__": "CodeReference", "label": "inference/model.py L429", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L429", "path": "inference/model.py", "line": 429, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L475", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L475", "path": "inference/model.py", "line": 475, "end_line": null}], "formula": null, "drawing": null}}, "562282282": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 562282282, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "712320119": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 712320119, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "930835929": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 930835929, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1094667341": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1094667341, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "751314649": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 751314649, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{d}}] = u[i_{\\bar{d}}], \\qquad y[|\\bar{d}| + i_{z}] = r[i_{z}], \\qquad |d| = |\\bar{d}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1813682219": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1813682219, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\hat{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "226550526": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 226550526, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "306323196": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 306323196, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "681129906": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 681129906, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1529155389": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1529155389, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-126})", "description": "Keeps the scale of a group at or above 2^-126, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-126 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L165", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L165", "path": "inference/kernel.py", "line": 165, "end_line": null}], "formula": "y = \\max(x, 2^{-126})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1634102166": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1634102166, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "645594142": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 645594142, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1508742795": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1508742795, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "544185802": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 544185802, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "700365590": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 700365590, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1292367284": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1292367284, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}] = x[i_{\\hat{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "660349806": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 660349806, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1484937847": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1484937847, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "904592488": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 904592488, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "957454559": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 957454559, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "224774189": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 224774189, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1821701682": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1821701682, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "473291465": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 473291465, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{d}}] = u[i_{\\bar{d}}], \\qquad y[|\\bar{d}| + i_{z}] = r[i_{z}], \\qquad |d| = |\\bar{d}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "711633062": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 711633062, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\hat{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "351247348": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 351247348, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "804256105": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 804256105, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1551634181": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1551634181, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1695743224": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1695743224, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-126})", "description": "Keeps the scale of a group at or above 2^-126, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-126 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L165", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L165", "path": "inference/kernel.py", "line": 165, "end_line": null}], "formula": "y = \\max(x, 2^{-126})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1566596659": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1566596659, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "310551720": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 310551720, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "708876119": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 708876119, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1231821024": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1231821024, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1404768491": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1404768491, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "866867178": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 866867178, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}] = x[i_{\\hat{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1752193106": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1752193106, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2034372932": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2034372932, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{b}, i_{r|b}] = x[i_{b} - i_{r|b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "277341155": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 277341155, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. The input does not carry every axis read by the row, so one copy of the input is written for every position of the missing axes. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|a|_{2}\\, i_{b} + i_{a} - 1 + |a|_{2}] = x[i_{b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1867449027": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1867449027, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "(\\lvert d \\rvert \\lvert i \\rvert)^{-1/2} x", "description": "This map multiplies the weight of every indexer head by 1/64. The released indexer multiplies by one over the square root of the key width, which is 128, and by one over the square root of the head count, which is 32. The model drawn without the omitted mechanisms folds this factor into the weight `w^{I}`. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L513", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L513", "path": "inference/model.py", "line": 513, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L555", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L555", "path": "inference/model.py", "line": 555, "end_line": null}], "formula": "y = x (|d|_{128} |i|_{32})^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2104666459": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2104666459, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the positions of the largest scores along the axis consumed and hands out no score, because nothing reads a score after the selection. Each result is a natural number, one per slot of the result's axis. A slot with no entry to hold reads the unit.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L579", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L579", "path": "inference/model.py", "line": 578, "end_line": 579}, {"__type__": "CodeReference", "label": "inference/model.py L607", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L607", "path": "inference/model.py", "line": 607, "end_line": null}], "formula": "\\{p[i_{s|x}]\\} = \\{i_{r|x} : s[i_{r|x}] \\text{ is among the } |s|_{512} \\text{ largest of } s\\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "722369367": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 722369367, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in INT64 and writes it in INT32. The written format holds an element in 32 bits against the 64 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{INT64} \\to \\mathtt{INT32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1127257566": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1127257566, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1482446053": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1482446053, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1348463247": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1348463247, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1616268353": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1616268353, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1532950353": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1532950353, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "687147876": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 687147876, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{E}, i_{y}] = x[|y|_{16}\\, i_{E} + i_{y}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1306098677": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1306098677, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "98207975": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 98207975, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "770153142": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 770153142, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "68924726": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 68924726, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-9})", "description": "Keeps the scale of a group at or above 2^-9, the smallest number above zero held by the E4M3 form, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-9 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L161-L162", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L161-L162", "path": "inference/kernel.py", "line": 161, "end_line": 162}], "formula": "y = \\max(x, 2^{-9})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "837945426": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 837945426, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1803741347": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1803741347, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "925201191": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 925201191, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1505994628": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1505994628, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "913874757": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 913874757, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "480847750": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 480847750, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|y|_{16}\\, i_{E} + i_{y}] = x[i_{E}, i_{y}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "846847051": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 846847051, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "703781183": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 703781183, "_name": {"__type__": "DynamicName", "body": "r|b", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 774326000}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__ref__": 662338339}}, "1108894740": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1108894740, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{b}, i_{r|b}] = x[i_{b} - i_{r|b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1580717866": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1580717866, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}, "shift": {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}]}, "extent": null}, "874307367": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 874307367, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. The input does not carry every axis read by the row, so one copy of the input is written for every position of the missing axes. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|a|_{2}\\, i_{b} + i_{a} - 1 + |a|_{2}] = x[i_{b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1131549985": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1131549985, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "974378722": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 974378722, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Entry Gather}", "description": "The gather reads the latents of the selected entries. The compressed entries are read back from every entry and merged onto the queries the way the indexer reads its keys. One latent is read per slot at the distance held by the selection for that slot.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": null, "drawing": null}}, "923802388": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 923802388, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation}", "description": "Writes each input onto its own run of the concatenated axis. The second run comes after the first. The size of that axis is therefore the sum of the sizes of the parts. Every other axis is carried through unchanged.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L778", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L778", "path": "inference/model.py", "line": 778, "end_line": null}], "formula": "y = x^{(1)} \\,\\Vert\\, x^{(2)}, \\quad |t| = |w| + |s|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "583181983": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 583181983, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2124677647": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2124677647, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1383171842": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1383171842, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1670016142": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1670016142, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "568884023": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 568884023, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots, the selected entries and the learned sink. The two kinds of latent are laid end to end along one slot axis. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "1231629490": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1231629490, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "744828180": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 744828180, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1023609338": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1023609338, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1262850148": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1262850148, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1008576951": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1008576951, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1427437243": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1427437243, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "743936849": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 743936849, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1798063302": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1798063302, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Full Attention}", "description": "The Full layer of an encoder group compresses every two consecutive tokens of its hidden state into one entry and makes one indexer key from each entry. The layer rotates and rounds both as the released caches hold them. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. The layer scores every entry reached by each query and keeps the 512 best. The layer then attends over the window and the kept entries under one softmax. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection. The entries and the selection are stored on the tape, which holds what one layer leaves for another. The Reuse layers of the same group read the entries and the selection there.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L739-L763", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L739-L763", "path": "inference/model.py", "line": 739, "end_line": 763}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}, {"__type__": "CodeReference", "label": "inference/model.py L1166-L1176", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1176", "path": "inference/model.py", "line": 1166, "end_line": 1176}], "formula": null, "drawing": null}}, "1487571524": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1487571524, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2051253512": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2051253512, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1713758829": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1713758829, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "1723850802": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1723850802, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1764058977": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1764058977, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "472798433": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 472798433, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "424442733": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 424442733, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "394422009": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 394422009, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1373348834": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1373348834, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1860466856": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1860466856, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "190619322": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 190619322, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1361346939": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1361346939, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "402943137": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 402943137, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1977236111": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1977236111, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1732720453": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1732720453, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1054348125": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1054348125, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1876963071": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1876963071, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "348429165": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 348429165, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "395373188": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 395373188, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "848192552": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 848192552, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1496637430": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1496637430, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1570615232": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1570615232, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2040465009": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2040465009, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1973130624": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1973130624, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1534596672": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1534596672, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "326469448": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 326469448, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2066107785": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2066107785, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "859113014": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 859113014, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "987211272": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 987211272, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1749817785": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1749817785, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1561229921": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1561229921, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1495270183": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1495270183, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "22464021": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 22464021, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1435023137": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1435023137, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "989550798": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 989550798, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "166632108": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 166632108, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1289614943": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1289614943, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1845328088": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1845328088, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "1768125654": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1768125654, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "236464602": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 236464602, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "94360803": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 94360803, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "662324187": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 662324187, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2045383558": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2045383558, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1994897626": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1994897626, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1940849049": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1940849049, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "47379085": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 47379085, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1425451196": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1425451196, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1705855416": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1705855416, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "359716953": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 359716953, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1465099631": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1465099631, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "584201354": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 584201354, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1066395770": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1066395770, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "261395537": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 261395537, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "524772590": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 524772590, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "635168157": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 635168157, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "727472696": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 727472696, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1429444763": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1429444763, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1967039422": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1967039422, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1533521062": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1533521062, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2071685446": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2071685446, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "434352578": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 434352578, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "365652868": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 365652868, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1319854359": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1319854359, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1243203156": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1243203156, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "327485532": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 327485532, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1595589340": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1595589340, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "772605348": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 772605348, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1850294511": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1850294511, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "612145333": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 612145333, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1561354489": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1561354489, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "189340970": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 189340970, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "406409602": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 406409602, "_name": {"__type__": "DynamicName", "body": "r|b", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 774326000}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__ref__": 662338339}}, "1831698068": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1831698068, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{b}, i_{r|b}] = x[i_{b} - i_{r|b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1263024188": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1263024188, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 662338339}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}, "shift": {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 2067311870}]}]}, "extent": null}, "396137204": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 396137204, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. The input does not carry every axis read by the row, so one copy of the input is written for every position of the missing axes. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|a|_{2}\\, i_{b} + i_{a} - 1 + |a|_{2}] = x[i_{b}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1651697958": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1651697958, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1213939107": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1213939107, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Entry Gather}", "description": "The gather reads the latents of the selected entries. The compressed entries are read back from every entry and merged onto the queries the way the indexer reads its keys. One latent is read per slot at the distance held by the selection for that slot.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": null, "drawing": null}}, "1540335988": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1540335988, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation}", "description": "Writes each input onto its own run of the concatenated axis. The second run comes after the first. The size of that axis is therefore the sum of the sizes of the parts. Every other axis is carried through unchanged.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L778", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L778", "path": "inference/model.py", "line": 778, "end_line": null}], "formula": "y = x^{(1)} \\,\\Vert\\, x^{(2)}, \\quad |t| = |w| + |s|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "602592902": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 602592902, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "720690612": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 720690612, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "607071637": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 607071637, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1777934808": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1777934808, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "658869866": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 658869866, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots, the selected entries and the learned sink. The two kinds of latent are laid end to end along one slot axis. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "2088056330": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2088056330, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1917009067": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1917009067, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2089213423": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2089213423, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "525091851": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 525091851, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1223568801": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1223568801, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1551961495": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1551961495, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "950774583": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 950774583, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "415189475": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 415189475, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reuse Attention}", "description": "A Reuse layer reads the entries and the selection from the tape, which holds what an earlier layer left for it. The layer therefore compresses no tokens and scores no entries. The entries arrive rotated and rounded. The layer computes its own query and window latents from its own hidden state. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. The layer reads the latents of the selected entries and attends over the window and those entries under one softmax. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection.", "fill_color": "#B8D8CE", "references": [{"__type__": "CodeReference", "label": "inference/model.py L1166-L1176", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1176", "path": "inference/model.py", "line": 1166, "end_line": 1176}, {"__type__": "CodeReference", "label": "inference/model.py L1180", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1180", "path": "inference/model.py", "line": 1180, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L762-L763", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L762-L763", "path": "inference/model.py", "line": 762, "end_line": 763}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}], "formula": null, "drawing": null}}, "664871163": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 664871163, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "921677816": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 921677816, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "287243756": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 287243756, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "24080531": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 24080531, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "125833732": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 125833732, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2018381300": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2018381300, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "202268705": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 202268705, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "476910351": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 476910351, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "620361200": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 620361200, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1627925086": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1627925086, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "325157539": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 325157539, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1107874271": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1107874271, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1085663578": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1085663578, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1587985339": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1587985339, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1639822141": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1639822141, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1619912639": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1619912639, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "168184036": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 168184036, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "690807969": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 690807969, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1977016297": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1977016297, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "830787826": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 830787826, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "819733816": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 819733816, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1480715581": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1480715581, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "560614621": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 560614621, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1776678308": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1776678308, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1592594036": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1592594036, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1980341966": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1980341966, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "118402732": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 118402732, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1745711812": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1745711812, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1419688920": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1419688920, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2001069412": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2001069412, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "619614411": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 619614411, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2099144851": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2099144851, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1534835245": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1534835245, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "850296110": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 850296110, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "254939139": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 254939139, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1024347189": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1024347189, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1582282131": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1582282131, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1395338646": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1395338646, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "401552601": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 401552601, "_name": null}, "repetition": {"__type__": "Integer", "_value": 5}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reuse Layer}", "description": "One of the five Reuse layers of the third encoder group: a Reuse attention sublayer reading member 2 of the entries and of the selection, written by the Full layer of the third group, and a mixture sublayer, each inside the four-stream residual.", "fill_color": "#F3F3F4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L27-L28", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L27-L28", "path": "inference/config.json", "line": 27, "end_line": 28}, {"__type__": "CodeReference", "label": "inference/model.py L1166-L1170", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1170", "path": "inference/model.py", "line": 1166, "end_line": 1170}], "formula": null, "drawing": null}}, "1598645706": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1598645706, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Third Encoder Group}", "description": "The third encoder group, which is layers 14 to 19: a Full layer followed by five Reuse layers, as in the two groups before it. The group is drawn on its own because Engram is applied to the residual before layer 14, which is the first layer of this group. A step that runs before the third group and before no other group cannot stand inside a block that repeats the same body three times, so the third group cannot be an iteration of that block. The two groups before it write the members 0 and 1 of the entries and of the selection, one member per iteration. This group writes and reads member 2, which is named by the constant 2 because no counter runs here.", "fill_color": "#EFEFF4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L27-L28", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L27-L28", "path": "inference/config.json", "line": 27, "end_line": 28}, {"__type__": "CodeReference", "label": "inference/config.json L43", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L43", "path": "inference/config.json", "line": 43, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L1262-L1263", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1262-L1263", "path": "inference/model.py", "line": 1262, "end_line": 1263}], "formula": null, "drawing": null}}, "2081341673": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2081341673, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1806938848": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1806938848, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1755990983": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1755990983, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "195114074": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 195114074, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1937923499": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1937923499, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1801524080": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1801524080, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1421844988": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1421844988, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "588801002": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 588801002, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "279651682": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 279651682, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1592011015": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1592011015, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "642521413": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 642521413, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "483473516": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 483473516, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2125952280": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2125952280, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "139056074": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 139056074, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1017011930": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1017011930, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "354967233": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 354967233, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1102592178": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1102592178, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1079758383": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1079758383, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1123719596": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1123719596, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1993856775": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1993856775, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "556102158": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 556102158, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1239295794": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1239295794, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "151178230": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 151178230, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "877509702": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 877509702, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1044200420": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1044200420, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "640332448": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 640332448, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "673472118": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 673472118, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "318568844": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 318568844, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1045661385": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1045661385, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2129130172": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2129130172, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "89574878": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 89574878, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2078398497": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 2078398497, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "1288718315": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1288718315, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1653210186": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1653210186, "_name": {"__type__": "DynamicName", "body": "B", "subscript": null, "settings": null, "code_form": "decoder_entries", "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "1557146755": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1557146755, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "This view is a renaming with stride one. Under the ratio-1 compressor each token is one entry, so entry i_B reads token i_B.", "fill_color": null, "references": null, "formula": "y[i_{B}] = x[i_{B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1320215207": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1320215207, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "326770741": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 326770741, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "785619145": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 785619145, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Compressor}", "description": "In the ratio-1 compressor each token is one entry. The token is projected to the latent width and normalised.", "fill_color": "#B8D8CE", "references": [{"__type__": "CodeReference", "label": "inference/model.py L429", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L429", "path": "inference/model.py", "line": 429, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L475", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L475", "path": "inference/model.py", "line": 475, "end_line": null}], "formula": null, "drawing": null}}, "1115790765": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1115790765, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "814222801": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 814222801, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "830663497": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 830663497, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1492582197": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1492582197, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1324835517": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1324835517, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{d}}] = u[i_{\\bar{d}}], \\qquad y[|\\bar{d}| + i_{z}] = r[i_{z}], \\qquad |d| = |\\bar{d}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "352677900": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 352677900, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Rotary Embedding of the Decoder Indexer Keys}", "description": "Each indexer key of the decoder is made from one compressed entry, which stands for one token. The key is rotated by the position of that token. The last 64 of the 128 channels of each vector are taken two at a time as points in a plane. The rotation turns each point about the origin. The other 64 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L537-L545", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L537-L545", "path": "inference/model.py", "line": 537, "end_line": 545}, {"__type__": "CodeReference", "label": "inference/model.py L392-L406", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L406", "path": "inference/model.py", "line": 392, "end_line": 406}], "formula": "\\begin{gathered} y[i_{B}, |\\bar{d}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{B}, |\\bar{d}| + 2 i_{t} + 1] = F[i_{B}, i_{t}]\\, \\big(v[i_{B}, |\\bar{d}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{B}, |\\bar{d}| + 2 i_{t} + 1]\\big) \\\\ y[i_{B}, i_{\\bar{d}}] = v[i_{B}, i_{\\bar{d}}] \\end{gathered}", "drawing": null}}, "1772608070": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1772608070, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\hat{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1028132146": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1028132146, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1093826417": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1093826417, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1247897351": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1247897351, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "240686026": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 240686026, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-126})", "description": "Keeps the scale of a group at or above 2^-126, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-126 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L165", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L165", "path": "inference/kernel.py", "line": 165, "end_line": null}], "formula": "y = \\max(x, 2^{-126})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "322363426": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 322363426, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1861428396": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1861428396, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1621718819": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1621718819, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "937163635": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 937163635, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1282488415": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1282488415, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1623365093": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1623365093, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}] = x[i_{\\hat{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "443658084": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 443658084, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "291002998": {"__type__": "TapeSlot", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "TapeSlot"}, "_id": 291002998, "_name": {"__type__": "DynamicName", "body": "\\mathrm{ik}", "subscript": {"__type__": "DynamicName", "body": "B", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": null, "code_form": null, "exponent": null}}}, "1878819435": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1878819435, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1276008677": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1276008677, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1804297791": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1804297791, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1684497926": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1684497926, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "673198769": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 673198769, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "114095653": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 114095653, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{d}}] = u[i_{\\bar{d}}], \\qquad y[|\\bar{d}| + i_{z}] = r[i_{z}], \\qquad |d| = |\\bar{d}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1451734723": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1451734723, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\hat{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "103419874": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 103419874, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1641825190": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1641825190, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "389022862": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 389022862, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2006086595": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2006086595, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-126})", "description": "Keeps the scale of a group at or above 2^-126, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-126 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L165", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L165", "path": "inference/kernel.py", "line": 165, "end_line": null}], "formula": "y = \\max(x, 2^{-126})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "62135158": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 62135158, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "720144560": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 720144560, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1599561773": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1599561773, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2097354982": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2097354982, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1829184405": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1829184405, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1438302153": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1438302153, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}] = x[i_{\\hat{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2082846268": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2082846268, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1394228890": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1394228890, "_name": {"__type__": "DynamicName", "body": "r|B", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 1653210186}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "2034888149": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2034888149, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{B}, i_{r|B}] = x[i_{B} - i_{r|B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "75071956": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 75071956, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "268478221": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 268478221, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[i_{B}] = x[i_{B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1389821898": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1389821898, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "(\\lvert d \\rvert \\lvert i \\rvert)^{-1/2} x", "description": "This map multiplies the weight of every indexer head by 1/64. The released indexer multiplies by one over the square root of the key width, which is 128, and by one over the square root of the head count, which is 32. The model drawn without the omitted mechanisms folds this factor into the weight `w^{I}`. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L513", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L513", "path": "inference/model.py", "line": 513, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L555", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L555", "path": "inference/model.py", "line": 555, "end_line": null}], "formula": "y = x (|d|_{128} |i|_{32})^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1211314350": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1211314350, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Indexer Scores}", "description": "One query is scored against every compressed entry reached by the query. The entries read by the attention are chosen from the scores. Each of the 32 indexer heads takes the product of its rotated and rounded indexer query with the key of the entry. A negative product is set to zero. The 32 results are summed under one weight per head, which is read off the hidden state of the token. The weights are multiplied by one over the square root of the key width times the head count, which is 1/64 in the released model. The released code computes the product with an ordinary 16-bit multiply over keys and queries that were rounded to four bits and written back as 16-bit numbers. The score is therefore the one given by a four-bit multiply. The reference implementation takes neither the smaller key cache nor the faster scoring.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L513", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L513", "path": "inference/model.py", "line": 513, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L515", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L515", "path": "inference/model.py", "line": 515, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L555-L557", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L555-L557", "path": "inference/model.py", "line": 555, "end_line": 557}], "formula": null, "drawing": null}}, "1533904032": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1533904032, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Lightning Indexer}", "description": "The lightning indexer scores every compressed entry reached by a query, so that the attention reads the best-scoring entries alone. The indexer query of every head is projected from the low-rank query latent, rotated by the position of its token and rounded to the four-bit FP4 format. The shared indexer keys arrive rotated and rounded. The keys are read back from every entry and merged onto the queries, so each query holds the keys of the entries reached by that query. The scoring box scores one query against those keys. The rounding makes the scores equal those of a system that keeps the keys as four-bit numbers and scores them on the four-bit hardware path. The released code is a reference implementation. It keeps the rounded keys and queries as 16-bit numbers and scores in 16-bit, so it takes neither the smaller cache nor the faster scoring.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L550-L552", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L550-L552", "path": "inference/model.py", "line": 550, "end_line": 552}, {"__type__": "CodeReference", "label": "inference/model.py L563-L565", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L563-L565", "path": "inference/model.py", "line": 563, "end_line": 565}, {"__type__": "CodeReference", "label": "inference/model.py L567", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L567", "path": "inference/model.py", "line": 567, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L577-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L577-L580", "path": "inference/model.py", "line": 577, "end_line": 580}], "formula": null, "drawing": null}}, "1695623199": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1695623199, "_name": {"__type__": "DynamicName", "body": "s|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "512", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 454012108}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "2026529264": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2026529264, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the positions of the largest scores along the axis consumed and hands out no score, because nothing reads a score after the selection. Each result is a natural number, one per slot of the result's axis. A slot with no entry to hold reads the unit.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L579", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L579", "path": "inference/model.py", "line": 578, "end_line": 579}, {"__type__": "CodeReference", "label": "inference/model.py L607", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L607", "path": "inference/model.py", "line": 607, "end_line": null}], "formula": "\\{p[i_{s|x}]\\} = \\{i_{r|x} : s[i_{r|x}] \\text{ is among the } |s|_{512} \\text{ largest of } s\\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1397108363": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1397108363, "_name": {"__type__": "DynamicName", "body": "P", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": "blocks_size", "exponent": null}}}, "9883567": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 9883567, "_name": {"__type__": "DynamicName", "body": "u", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "block_size_size", "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1061213771": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1061213771, "_name": {"__type__": "DynamicName", "body": "P|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__ref__": 1397108363}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 9883567}]}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "145414426": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 145414426, "_name": {"__type__": "DynamicName", "body": "u|x,P", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 9883567}, "guides": [{"__ref__": 777392202}, {"__ref__": 1061213771}], "guide_strides": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 9883567}]}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "1782368186": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1782368186, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "blk", "description": "Cuts the distances reached by a query into blocks of |u| consecutive distances. Offset i_u of block i_P reads distance |u| i_P + i_u. The candidate pool keeps the best score of each block.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L599", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L599", "path": "inference/model.py", "line": 599, "end_line": null}], "formula": "y[i_{P|x}, i_{u|x,P}] = x[|u|_{8}\\, i_{P|x} + i_{u|x,P}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1643587181": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1643587181, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Positive Infinity}", "description": "Positive infinity is larger than every score. Written at block 0 and added to the block scores of a query, it puts the block of the newest entries ahead of every other block, so the Top-2048 over the blocks keeps that block.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L602-L605", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L602-L605", "path": "inference/model.py", "line": 602, "end_line": 605}, {"__type__": "CodeReference", "label": "inference/model.py L607", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L607", "path": "inference/model.py", "line": 607, "end_line": null}], "formula": "J = \\infty", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1106466778": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1106466778, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pin", "description": "Writes the input at the position named by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[0] = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2136810350": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2136810350, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pinned Newest Block}", "description": "The block of the eight newest entries is always kept as a candidate. Positive infinity is written at block 0 and added to the block scores. Every other block keeps its score. No score is larger than infinity, so the Top-2048 that follows keeps block 0. The released code numbers its blocks by absolute entry. The newest block of the released code is still filling and holds between one and eight entries, so an older block that is full could score higher. The released code pins the newest block so that a full older block cannot displace it. The model drawn here numbers the blocks by distance back from the newest entry reached by a query, so block 0 holds the newest entries of every query. Once a query has reached eight entries, block 0 holds eight of them. Every entry pinned by the released code is therefore in block 0, beside up to seven older entries.", "fill_color": "#FBF3D5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L602-L605", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L602-L605", "path": "inference/model.py", "line": 602, "end_line": 605}, {"__type__": "CodeReference", "label": "inference/model.py L597-L599", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L597-L599", "path": "inference/model.py", "line": 597, "end_line": 599}, {"__type__": "CodeReference", "label": "inference/model.py L607", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L607", "path": "inference/model.py", "line": 607, "end_line": null}], "formula": "\\mathrm{pin}(b)[i_{P}] = b[i_{P}] + J[i_{P}], \\qquad J[0] = \\infty, \\qquad J[i_{P}] = 0 \\text{ for } i_{P} \\neq 0", "drawing": null}}, "1259870718": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1259870718, "_name": {"__type__": "DynamicName", "body": "p", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "2048", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1964091467": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1964091467, "_name": {"__type__": "DynamicName", "body": "p|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "2048", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1259870718}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 9883567}]}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "2123138890": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2123138890, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the positions of the largest scores along the axis consumed and hands out no score, because nothing reads a score after the selection. Each result is a natural number, one per slot of the result's axis. A slot with no entry to hold reads the unit.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L579", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L579", "path": "inference/model.py", "line": 578, "end_line": 579}, {"__type__": "CodeReference", "label": "inference/model.py L607", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L607", "path": "inference/model.py", "line": 607, "end_line": null}], "formula": "\\{p[i_{p|x}]\\} = \\{i_{P|x} : s[i_{P|x}] \\text{ is among the } |p|_{2048} \\text{ largest of } s\\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "634102213": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 634102213, "_name": {"__type__": "DynamicName", "body": "P", "subscript": null, "settings": null, "code_form": "blocks", "exponent": null}}, "_size": {"__ref__": 1397108363}}, "663751963": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 663751963, "_name": {"__type__": "DynamicName", "body": "u", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "block_size", "exponent": {"__type__": "DynamicName", "body": "8", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 9883567}}, "2006286199": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2006286199, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "blk", "description": "Turns the number of a selected block into the positions of the entries held by the block. Block i_P holds one entry per offset i_u, at position |u| i_P + i_u. The result therefore carries the offset axis.", "fill_color": null, "references": null, "formula": "i_{B} = |u|\\, i_{P} + i_{u}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "833815902": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 833815902, "_name": {"__type__": "DynamicName", "body": "C", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "candidates_size", "exponent": {"__type__": "DynamicName", "body": "16384", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "1268809917": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 1268809917, "_name": {"__type__": "DynamicName", "body": "C", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": "candidates", "exponent": {"__type__": "DynamicName", "body": "16384", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 833815902}}, "177481649": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 177481649, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "cand", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|u|_{8}\\, i_{p|x} + i_{u}] = x[i_{p|x}, i_{u}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "281496344": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 281496344, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in INT64 and writes it in INT32. The written format holds an element in 32 bits against the 64 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{INT64} \\to \\mathtt{INT32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "114731108": {"__type__": "TapeSlot", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "TapeSlot"}, "_id": 114731108, "_name": {"__type__": "DynamicName", "body": "\\mathrm{pool}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "2046904689": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2046904689, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Candidate Pool}", "description": "The block turns one token's indexer scores into the distances each Reindex layer scores inside. The distances are split into blocks of eight. Each block keeps its best score. The block of the eight newest entries is pinned. The 2048 best blocks are kept. The eight distances of every kept block are laid out along the candidate axis.", "fill_color": "#F5E6D9", "references": [{"__type__": "CodeReference", "label": "inference/model.py L583-L610", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L583-L610", "path": "inference/model.py", "line": 583, "end_line": 610}, {"__type__": "CodeReference", "label": "inference/model.py L597-L599", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L597-L599", "path": "inference/model.py", "line": 597, "end_line": 599}, {"__type__": "CodeReference", "label": "inference/model.py L602-L605", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L602-L605", "path": "inference/model.py", "line": 602, "end_line": 605}, {"__type__": "CodeReference", "label": "inference/model.py L607", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L607", "path": "inference/model.py", "line": 607, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L609-L610", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L609-L610", "path": "inference/model.py", "line": 609, "end_line": 610}], "formula": null, "drawing": null}}, "482911146": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 482911146, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in INT64 and writes it in INT32. The written format holds an element in 32 bits against the 64 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{INT64} \\to \\mathtt{INT32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1050330992": {"__type__": "TapeSlot", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "TapeSlot"}, "_id": 1050330992, "_name": {"__type__": "DynamicName", "body": "\\mathrm{sel}", "subscript": {"__type__": "DynamicName", "body": "B", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": null, "code_form": null, "exponent": null}}}, "482772026": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 482772026, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "964017876": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 964017876, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "693678664": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 693678664, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1668707412": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1668707412, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1845775155": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1845775155, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1837559765": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1837559765, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Rotary Embedding of the Decoder Entries}", "description": "Each compressed entry of the decoder stands for one token. The entry is rotated by the position of that token. The last 64 of the 512 channels of each vector are taken two at a time as points in a plane. The rotation turns each point about the origin. The other 448 channels pass through unchanged. The circle in the box is the table of turns, which holds one turn for every position and every pair of channels. The indexer keys are made from the entries before the rotation. The attention of this layer and of every later layer of the decoder reads the entries after the rotation.", "fill_color": "#E6EEF5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L749-L758", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L749-L758", "path": "inference/model.py", "line": 749, "end_line": 758}, {"__type__": "CodeReference", "label": "inference/model.py L392-L406", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L392-L406", "path": "inference/model.py", "line": 392, "end_line": 406}], "formula": "\\begin{gathered} y[i_{B}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, y[i_{B}, |\\bar{z}| + 2 i_{t} + 1] = F[i_{B}, i_{t}]\\, \\big(v[i_{B}, |\\bar{z}| + 2 i_{t}] + \\mathrm{i}\\, v[i_{B}, |\\bar{z}| + 2 i_{t} + 1]\\big) \\\\ y[i_{B}, i_{\\bar{z}}] = v[i_{B}, i_{\\bar{z}}] \\end{gathered}", "drawing": null}}, "1765077251": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1765077251, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{E}, i_{y}] = x[|y|_{16}\\, i_{E} + i_{y}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1328972565": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1328972565, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1356145885": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1356145885, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "157811102": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 157811102, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1535081910": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1535081910, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-9})", "description": "Keeps the scale of a group at or above 2^-9, the smallest number above zero held by the E4M3 form, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-9 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L161-L162", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L161-L162", "path": "inference/kernel.py", "line": 161, "end_line": 162}], "formula": "y = \\max(x, 2^{-9})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "587290840": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 587290840, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1587125108": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1587125108, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "171767467": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 171767467, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "936257975": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 936257975, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "8361378": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 8361378, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1387124435": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1387124435, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|y|_{16}\\, i_{E} + i_{y}] = x[i_{E}, i_{y}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1087519437": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1087519437, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "378367793": {"__type__": "TapeSlot", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "TapeSlot"}, "_id": 378367793, "_name": {"__type__": "DynamicName", "body": "\\mathrm{ckv}", "subscript": {"__type__": "DynamicName", "body": "B", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": null, "code_form": null, "exponent": null}}}, "432828724": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 432828724, "_name": {"__type__": "DynamicName", "body": "r|B", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 1653210186}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "1005197669": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1005197669, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{B}, i_{r|B}] = x[i_{B} - i_{r|B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "323018765": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 323018765, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "354176676": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 354176676, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[i_{B}] = x[i_{B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1880704462": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1880704462, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "728312760": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 728312760, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Entry Gather}", "description": "The gather reads the latents of the selected entries. The compressed entries are read back from every entry and merged onto the queries the way the indexer reads its keys. One latent is read per slot at the distance held by the selection for that slot.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": null, "drawing": null}}, "1232458515": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1232458515, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation}", "description": "Writes each input onto its own run of the concatenated axis. The second run comes after the first. The size of that axis is therefore the sum of the sizes of the parts. Every other axis is carried through unchanged.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L778", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L778", "path": "inference/model.py", "line": 778, "end_line": null}], "formula": "y = x^{(1)} \\,\\Vert\\, x^{(2)}, \\quad |t| = |w| + |s|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1128811205": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1128811205, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1838108987": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1838108987, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "667031627": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 667031627, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1884121500": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1884121500, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "671103403": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 671103403, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots, the selected entries and the learned sink. The two kinds of latent are laid end to end along one slot axis. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "658479500": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 658479500, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1362998725": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1362998725, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "446710450": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 446710450, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1757196326": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1757196326, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "353618956": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 353618956, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "651740490": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 651740490, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "983766265": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 983766265, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1269844291": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1269844291, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Full Attention}", "description": "The Full layer of the decoder projects each token of its hidden state into one entry and makes one indexer key from each entry. The layer rotates and rounds both as the released caches hold them. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. The layer scores every entry reached by each query and keeps the 512 best. The layer then attends over the window and the kept entries under one softmax. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection. The layer also keeps the 2048 best blocks of eight entries as the candidate pool. The newest block is always among them. The layer stores the entries, the keys, the pool and the selection on the tape, which holds them for every later decoder layer.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L739-L763", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L739-L763", "path": "inference/model.py", "line": 739, "end_line": 763}, {"__type__": "CodeReference", "label": "inference/model.py L569-L575", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L569-L575", "path": "inference/model.py", "line": 569, "end_line": 575}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}, {"__type__": "CodeReference", "label": "inference/model.py L1166-L1176", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1176", "path": "inference/model.py", "line": 1166, "end_line": 1176}], "formula": null, "drawing": null}}, "1109976446": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1109976446, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1716648920": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1716648920, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "969708795": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 969708795, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "1271546785": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1271546785, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1738199092": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1738199092, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "73824524": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 73824524, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1941201686": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1941201686, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "21455696": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 21455696, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1542789657": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1542789657, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1814678553": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1814678553, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2117105084": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2117105084, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "573189088": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 573189088, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1305068497": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1305068497, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "154397890": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 154397890, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "440405708": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 440405708, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "222867938": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 222867938, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1907141396": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1907141396, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1810973754": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1810973754, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "359447004": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 359447004, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "302058176": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 302058176, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1872496370": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1872496370, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "339687908": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 339687908, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "85423586": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 85423586, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "901418529": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 901418529, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "889161099": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 889161099, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1294078526": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1294078526, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1167535948": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1167535948, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1325276210": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1325276210, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "933696875": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 933696875, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1057023643": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1057023643, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "854881906": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 854881906, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1857544712": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1857544712, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1982518956": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1982518956, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1558212562": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1558212562, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "126307839": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 126307839, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1235208371": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1235208371, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1079728459": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1079728459, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1046191607": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1046191607, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "1055742195": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1055742195, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "234710216": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 234710216, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1588718640": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1588718640, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1438854861": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1438854861, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2053972306": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2053972306, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "574576833": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 574576833, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1754422891": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1754422891, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "792942731": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 792942731, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1651796171": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1651796171, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "920497946": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 920497946, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1853853613": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1853853613, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1166603162": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1166603162, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2058438074": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2058438074, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "12236898": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 12236898, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1479775281": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1479775281, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "558693900": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 558693900, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "222491822": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 222491822, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "296966002": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 296966002, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "588281363": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 588281363, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "492066828": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 492066828, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1773506187": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1773506187, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1031216167": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1031216167, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1570998651": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1570998651, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1194325895": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1194325895, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "166093394": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 166093394, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2118602141": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2118602141, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "900175072": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 900175072, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1970078746": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1970078746, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "826954011": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 826954011, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2032053637": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2032053637, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1489946038": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1489946038, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "54414998": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 54414998, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "1361853986": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1361853986, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2147295445": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 2147295445, "_name": {"__type__": "DynamicName", "body": "r|B", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 1653210186}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "2078340038": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2078340038, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{B}, i_{r|B}] = x[i_{B} - i_{r|B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "190663846": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 190663846, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "317655071": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 317655071, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[i_{B}] = x[i_{B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1471074630": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1471074630, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1201548377": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1201548377, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Entry Gather}", "description": "The gather reads the latents of the selected entries. The compressed entries are read back from every entry and merged onto the queries the way the indexer reads its keys. One latent is read per slot at the distance held by the selection for that slot.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": null, "drawing": null}}, "1277306869": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1277306869, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation}", "description": "Writes each input onto its own run of the concatenated axis. The second run comes after the first. The size of that axis is therefore the sum of the sizes of the parts. Every other axis is carried through unchanged.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L778", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L778", "path": "inference/model.py", "line": 778, "end_line": null}], "formula": "y = x^{(1)} \\,\\Vert\\, x^{(2)}, \\quad |t| = |w| + |s|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1079853878": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1079853878, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "437298516": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 437298516, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1600314139": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1600314139, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "806242071": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 806242071, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "678113622": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 678113622, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots, the selected entries and the learned sink. The two kinds of latent are laid end to end along one slot axis. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "2101151165": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2101151165, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "542512769": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 542512769, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1837778475": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1837778475, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1715697384": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1715697384, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1993833832": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1993833832, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "730802015": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 730802015, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1296710914": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1296710914, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2012807401": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2012807401, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reuse Attention}", "description": "A Reuse layer reads the entries and the selection from the tape, which holds what an earlier layer left for it. The layer therefore compresses no tokens and scores no entries. The entries arrive rotated and rounded. The layer computes its own query and window latents from its own hidden state. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. The layer reads the latents of the selected entries and attends over the window and those entries under one softmax. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection.", "fill_color": "#B8D8CE", "references": [{"__type__": "CodeReference", "label": "inference/model.py L1166-L1176", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1176", "path": "inference/model.py", "line": 1166, "end_line": 1176}, {"__type__": "CodeReference", "label": "inference/model.py L1180", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1180", "path": "inference/model.py", "line": 1180, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L762-L763", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L762-L763", "path": "inference/model.py", "line": 762, "end_line": 763}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}], "formula": null, "drawing": null}}, "45453915": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 45453915, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "55805327": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 55805327, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "374152582": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 374152582, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "344438980": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 344438980, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "124121295": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 124121295, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "34524970": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 34524970, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "671881358": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 671881358, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1034203203": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1034203203, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "237638215": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 237638215, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1959802137": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1959802137, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1768614888": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1768614888, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1687856698": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1687856698, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "316715274": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 316715274, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2071583129": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2071583129, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1603386714": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1603386714, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1475875656": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1475875656, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1670456729": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1670456729, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "372497703": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 372497703, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1673234456": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1673234456, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "725858608": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 725858608, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "260103834": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 260103834, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "309311570": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 309311570, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1696206085": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1696206085, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "628925550": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 628925550, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "972961105": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 972961105, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "852113373": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 852113373, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "29122101": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 29122101, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "149908305": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 149908305, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1538836246": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1538836246, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "294842487": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 294842487, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1709751211": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1709751211, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1082554513": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1082554513, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1400816842": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1400816842, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "621422887": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 621422887, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1469496832": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1469496832, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1004960373": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1004960373, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "789772136": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 789772136, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "339952228": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 339952228, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "287145830": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 287145830, "_name": null}, "repetition": {"__type__": "Integer", "_value": 3}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reuse Layer}", "description": "One of the three Reuse layers under the decoder's Full layer: a Reuse attention sublayer reading the entries and the selection written by that layer, and a mixture sublayer, each inside the four-stream residual.", "fill_color": "#F3F3F4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L27-L28", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L27-L28", "path": "inference/config.json", "line": 27, "end_line": 28}, {"__type__": "CodeReference", "label": "inference/model.py L1166-L1170", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1170", "path": "inference/model.py", "line": 1166, "end_line": 1170}], "formula": null, "drawing": null}}, "1682733140": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1682733140, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "46660753": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 46660753, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2039635623": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2039635623, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "843066693": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 843066693, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1190216838": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1190216838, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "133213170": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 133213170, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1390726711": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1390726711, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "776079857": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 776079857, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1253179404": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1253179404, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1478777232": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1478777232, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1789314188": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1789314188, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1480738335": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1480738335, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1725957783": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1725957783, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "187326743": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 187326743, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "551292411": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 551292411, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "904328899": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 904328899, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "439835104": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 439835104, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "851576806": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 851576806, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2023550518": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2023550518, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1231856600": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1231856600, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2152606": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2152606, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1249778365": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1249778365, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1072520906": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1072520906, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1204715483": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1204715483, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2096902558": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2096902558, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "659433345": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 659433345, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1530030910": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1530030910, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "492115116": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 492115116, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "146386394": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 146386394, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "643046060": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 643046060, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "714786929": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 714786929, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "196259415": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 196259415, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "757332449": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 757332449, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "106981378": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 106981378, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1377896999": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1377896999, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2022066351": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2022066351, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "347719303": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 347719303, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "112656838": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 112656838, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "84106018": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 84106018, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{d}}] = u[i_{\\bar{d}}], \\qquad y[|\\bar{d}| + i_{z}] = r[i_{z}], \\qquad |d| = |\\bar{d}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "311491011": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 311491011, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\hat{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "438404368": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 438404368, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "305083282": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 305083282, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "832932108": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 832932108, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 6", "description": "This map divides the largest magnitude of a group by 6, the largest number held by the E2M1 form. A group divided by this scale has its largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L131-L132", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L131-L132", "path": "inference/kernel.py", "line": 131, "end_line": 132}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x 6^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "489208514": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 489208514, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 2^{-126})", "description": "Keeps the scale of a group at or above 2^-126, so a group of zeros is not divided by zero. The released kernel keeps the largest magnitude at or above 6 times 2^-126 before it divides by 6, which is the same function. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L165", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L165", "path": "inference/kernel.py", "line": 165, "end_line": null}], "formula": "y = \\max(x, 2^{-126})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "60706018": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 60706018, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "393663376": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 393663376, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1874947529": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1874947529, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-6}^{6}", "description": "Keeps a divided channel between -6 and 6, the range of the E2M1 form. A scale rounded up to a power of two leaves every channel inside the range. A scale rounded through E4M3 can round down. The clamp then holds the largest channel at 6. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L171", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L171", "path": "inference/kernel.py", "line": 171, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-6}^{6}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1847866": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1847866, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E2M1}", "description": "Rounds each value to the nearest number held by the E2M1 form. The form has four bits: a sign, two exponent bits and one mantissa bit. The numbers are 0, 0.5, 1, 1.5, 2, 3, 4 and 6 with either sign. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = \\mathrm{round}_{\\mathrm{E2M1}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2111652837": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2111652837, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E2M1}", "description": "Reads each E2M1 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L167-L172", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L167-L172", "path": "inference/kernel.py", "line": 167, "end_line": 172}, {"__type__": "CodeReference", "label": "inference/convert.py L13-L15", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/convert.py#L13-L15", "path": "inference/convert.py", "line": 13, "end_line": 15}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "854622161": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 854622161, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\hat{E}} + i_{\\hat{y}}] = x[i_{\\hat{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "395400883": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 395400883, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1653828389": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1653828389, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{B}, i_{r|B}] = x[i_{B} - i_{r|B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1773500669": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1773500669, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[i_{B}] = x[i_{B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "313234043": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 313234043, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "(\\lvert d \\rvert \\lvert i \\rvert)^{-1/2} x", "description": "This map multiplies the weight of every indexer head by 1/64. The released indexer multiplies by one over the square root of the key width, which is 128, and by one over the square root of the head count, which is 32. The model drawn without the omitted mechanisms folds this factor into the weight `w^{I}`. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L513", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L513", "path": "inference/model.py", "line": 513, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L555", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L555", "path": "inference/model.py", "line": 555, "end_line": null}], "formula": "y = x (|d|_{128} |i|_{32})^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "129008273": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 129008273, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Lightning Indexer}", "description": "The lightning indexer scores every compressed entry reached by a query, so that the attention reads the best-scoring entries alone. The indexer query of every head is projected from the low-rank query latent, rotated by the position of its token and rounded to the four-bit FP4 format. The shared indexer keys arrive rotated and rounded. The keys are read back from every entry and merged onto the queries, so each query holds the keys of the entries reached by that query. The scoring box scores one query against those keys. The rounding makes the scores equal those of a system that keeps the keys as four-bit numbers and scores them on the four-bit hardware path. The released code is a reference implementation. It keeps the rounded keys and queries as 16-bit numbers and scores in 16-bit, so it takes neither the smaller cache nor the faster scoring.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L550-L552", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L550-L552", "path": "inference/model.py", "line": 550, "end_line": 552}, {"__type__": "CodeReference", "label": "inference/model.py L563-L565", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L563-L565", "path": "inference/model.py", "line": 563, "end_line": 565}, {"__type__": "CodeReference", "label": "inference/model.py L567", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L567", "path": "inference/model.py", "line": 567, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L577-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L577-L580", "path": "inference/model.py", "line": 577, "end_line": 580}], "formula": null, "drawing": null}}, "974054510": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 974054510, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1148834680": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1148834680, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the positions of the largest scores along the axis consumed and hands out no score, because nothing reads a score after the selection. Each result is a natural number, one per slot of the result's axis. A slot with no entry to hold reads the unit.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L579", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L579", "path": "inference/model.py", "line": 578, "end_line": 579}, {"__type__": "CodeReference", "label": "inference/model.py L607", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L607", "path": "inference/model.py", "line": 607, "end_line": null}], "formula": "\\{p[i_{s|x}]\\} = \\{i_{C} : s[i_{C}] \\text{ is among the } |s|_{512} \\text{ largest of } s\\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1088736842": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1088736842, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in INT64 and writes it in INT32. The written format holds an element in 32 bits against the 64 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{INT64} \\to \\mathtt{INT32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1370144125": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1370144125, "_name": {"__type__": "DynamicName", "body": "r|B", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 1653210186}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "1458167905": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1458167905, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{B}, i_{r|B}] = x[i_{B} - i_{r|B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "822342365": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 822342365, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "844956181": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 844956181, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[i_{B}] = x[i_{B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "175340687": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 175340687, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "523225846": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 523225846, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Entry Gather}", "description": "The gather reads the latents of the selected entries. The compressed entries are read back from every entry and merged onto the queries the way the indexer reads its keys. One latent is read per slot at the distance held by the selection for that slot.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": null, "drawing": null}}, "184420851": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 184420851, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation}", "description": "Writes each input onto its own run of the concatenated axis. The second run comes after the first. The size of that axis is therefore the sum of the sizes of the parts. Every other axis is carried through unchanged.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L778", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L778", "path": "inference/model.py", "line": 778, "end_line": null}], "formula": "y = x^{(1)} \\,\\Vert\\, x^{(2)}, \\quad |t| = |w| + |s|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "887545413": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 887545413, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1514596920": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1514596920, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "445146736": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 445146736, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1959188696": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1959188696, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "138745809": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 138745809, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots, the selected entries and the learned sink. The two kinds of latent are laid end to end along one slot axis. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "2118725060": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2118725060, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "334158217": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 334158217, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "57657454": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 57657454, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2049768762": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2049768762, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2129132298": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2129132298, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "437367122": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 437367122, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "58873396": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 58873396, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "951196658": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 951196658, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reindex Attention}", "description": "A Reindex layer reads the entries, the indexer keys and the candidate pool from the tape, which holds what the Full layer of the decoder left for later layers. All three arrive rotated and rounded. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. The layer scores its own rotated and rounded indexer queries against the entries of the pool alone and keeps the 512 best. The layer then attends over the window and the kept entries under one softmax. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection. The new selection is stored on the tape, where the Reuse layers of the same group read it.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L563-L565", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L563-L565", "path": "inference/model.py", "line": 563, "end_line": 565}, {"__type__": "CodeReference", "label": "inference/model.py L569-L575", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L569-L575", "path": "inference/model.py", "line": 569, "end_line": 575}, {"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}], "formula": null, "drawing": null}}, "452222473": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 452222473, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "294046117": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 294046117, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "474213553": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 474213553, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "1329062073": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1329062073, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2011695373": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2011695373, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1003534118": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1003534118, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2092220586": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2092220586, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "963566423": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 963566423, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "728848335": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 728848335, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "716215114": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 716215114, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2080262644": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2080262644, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1562932088": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1562932088, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "922839626": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 922839626, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1434451765": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1434451765, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1806147231": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1806147231, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1936605495": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1936605495, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "341872784": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 341872784, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "354316175": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 354316175, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "832636189": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 832636189, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1020151287": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1020151287, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1238623636": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1238623636, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "531820605": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 531820605, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "25387241": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 25387241, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1508358653": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1508358653, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1507489792": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1507489792, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1072726163": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1072726163, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "288512405": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 288512405, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "516290782": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 516290782, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1441015477": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1441015477, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1057077023": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1057077023, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "960948892": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 960948892, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "347436602": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 347436602, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1544683028": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1544683028, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "106939137": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 106939137, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "57212437": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 57212437, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1068062414": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1068062414, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1709535468": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1709535468, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1340619162": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1340619162, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "241005583": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 241005583, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1191391351": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1191391351, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "374076692": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 374076692, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "155874210": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 155874210, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1166944733": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1166944733, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "178087027": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 178087027, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "553525685": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 553525685, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1504494575": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1504494575, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "758111596": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 758111596, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "793460395": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 793460395, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1823581453": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1823581453, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1287422040": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1287422040, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1043259255": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1043259255, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "792220536": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 792220536, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1837228860": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1837228860, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "755977766": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 755977766, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1223227967": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1223227967, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "672653564": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 672653564, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1309921987": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1309921987, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "847564063": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 847564063, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Cuts the channels of one latent into consecutive groups of equal size. Channel i_y of group i_E reads channel |y| i_E + i_y of the latent. Every group is rounded with a scale of its own, so one large channel reduces the precision of its own group and of no other group.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L72", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L72", "path": "inference/kernel.py", "line": 72, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L156", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L156", "path": "inference/kernel.py", "line": 156, "end_line": null}], "formula": "y[i_{\\tilde{E}}, i_{\\hat{y}}] = x[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "818907372": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 818907372, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "576309722": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 576309722, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert x \\rvert", "description": "The magnitude of a channel is the channel without its sign. The largest magnitude of a group sets the scale of the group. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L74", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L74", "path": "inference/kernel.py", "line": 74, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L158", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L158", "path": "inference/kernel.py", "line": 158, "end_line": null}], "formula": "y = \\lvert x \\rvert", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "642289102": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 642289102, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\max(x, 10^{-4})", "description": "Keeps the largest magnitude of a group at or above 10^-4, so a group of zeros is not divided by zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L76", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L76", "path": "inference/kernel.py", "line": 76, "end_line": null}], "formula": "y = \\max(x, 10^{-4})", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "40690660": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 40690660, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / 448", "description": "This map divides the largest magnitude of a group by 448, the largest number held by the E4M3 form. A group divided by this scale has its largest channel at 448. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L44-L46", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L44-L46", "path": "inference/kernel.py", "line": 44, "end_line": 46}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}], "formula": "y = x 448^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1880856910": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1880856910, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\log_{2} x", "description": "This map takes the logarithm to base two of the scale. The ceiling of this number is the exponent of the smallest power of two at or above the scale. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}], "formula": "y = log_{2}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "696182075": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 696182075, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Ceiling}", "description": "Rounds a number up to the nearest whole number. Here the number is the logarithm to base two of a scale, so two to the power of the result is the smallest power of two at or above the scale. The released kernel reads the exponent bits of the scale and adds one where any mantissa bit is set. The package has no numeric for a ceiling or a floor, so this step is a generic operator.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L22-L27", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L22-L27", "path": "inference/kernel.py", "line": 22, "end_line": 27}, {"__type__": "CodeReference", "label": "inference/kernel.py L36-L37", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L36-L37", "path": "inference/kernel.py", "line": 36, "end_line": 37}, {"__type__": "CodeReference", "label": "inference/kernel.py L78", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L78", "path": "inference/kernel.py", "line": 78, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L166", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L166", "path": "inference/kernel.py", "line": 166, "end_line": null}], "formula": "\\lceil \\cdot \\rceil : R[\\,] \\to R[\\,], \\qquad \\lceil x \\rceil = \\min \\{ n \\in \\mathbb{Z} : n \\geq x \\}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1296715455": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1296715455, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-448}^{448}", "description": "Keeps a divided channel between -448 and 448, the range of the E4M3 form. The scale is rounded up, so every channel is already inside the range. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L85", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L85", "path": "inference/kernel.py", "line": 85, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-448}^{448}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "390227954": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 390227954, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to E4M3}", "description": "Rounds each value to the nearest number held by the E4M3 form. The form has eight bits: a sign, four exponent bits and three mantissa bits. The largest number is 448 and the smallest above zero is 2^-9. The operator names the datatype read and the datatype returned. The package writes no formula for the rounding.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = \\mathrm{round}_{\\mathrm{E4M3}}(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1448506997": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1448506997, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast from E4M3}", "description": "Reads each E4M3 number as a real. No value changes. The operations that follow are computed in reals, as the released kernel computes them in 32-bit floats.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L81-L86", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L81-L86", "path": "inference/kernel.py", "line": 81, "end_line": 86}, {"__type__": "CodeReference", "label": "inference/kernel.py L163", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L163", "path": "inference/kernel.py", "line": 163, "end_line": null}], "formula": "y = x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2139765997": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2139765997, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "qgrp", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[|\\hat{y}|_{32}\\, i_{\\tilde{E}} + i_{\\hat{y}}] = x[i_{\\tilde{E}}, i_{\\hat{y}}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2069040419": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2069040419, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1060190349": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 1060190349, "_name": {"__type__": "DynamicName", "body": "w|x", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "128", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 990851622}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "1579472843": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1579472843, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "win", "description": "The view is the sliding window. Slot i_w of query i_x reads the token i_w places before the query. A slot that reaches before the first token holds no value. The guarded axis w|x records which slots hold no value.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L410-L424", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L410-L424", "path": "inference/model.py", "line": 410, "end_line": 424}], "formula": "y[i_{x}, i_{w|x}] = x[i_{x} - i_{w|x}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "499599277": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 499599277, "_name": {"__type__": "DynamicName", "body": "r|B", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 1653210186}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}}, "565285220": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 565285220, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "back", "description": "Counts entries back from the entry named by a position. Distance i_r of entry i_b reads the entry i_r places before it. A distance that reaches before the first entry holds no value.", "fill_color": null, "references": null, "formula": "y[i_{B}, i_{r|B}] = x[i_{B} - i_{r|B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2006135640": {"__type__": "AffineSparseAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "AffineSparseAxis"}, "_id": 2006135640, "_name": {"__type__": "DynamicName", "body": "r|x", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "_size": {"__type__": "Multiplication", "content": [{"__ref__": 2067311870}, {"__ref__": 662338339}]}, "guides": [{"__ref__": 777392202}], "guide_strides": [{"__type__": "Integer", "_value": 1}], "stride": {"__type__": "Integer", "_value": -1}, "shift": {"__type__": "Integer", "_value": 0}, "extent": null}, "888916501": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 888916501, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "pos", "description": "Writes the input at each position to the output at the position given for it by the row of the formula. An output position written by no input position holds the unit of the datatype. An ordinary view reads in the other direction. It takes the output at each position from the input at the position named by its row.", "fill_color": null, "references": null, "formula": "y[i_{B}] = x[i_{B}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1724902817": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1724902817, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{IndexSelect}", "description": "Reads the payload at an index that arrives as data. The index is a natural number below the size of the axis it reads along. The operator is broadcast over every other axis, so one entry is read per index.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": "\\mathrm{IndexSelect}(i, p) = p[i]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1386873758": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1386873758, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Entry Gather}", "description": "The gather reads the latents of the selected entries. The compressed entries are read back from every entry and merged onto the queries the way the indexer reads its keys. One latent is read per slot at the distance held by the selection for that slot.", "fill_color": "#D9E7F5", "references": [{"__type__": "CodeReference", "label": "inference/model.py L578-L580", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L578-L580", "path": "inference/model.py", "line": 578, "end_line": 580}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}], "formula": null, "drawing": null}}, "500370804": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 500370804, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation}", "description": "Writes each input onto its own run of the concatenated axis. The second run comes after the first. The size of that axis is therefore the sum of the sizes of the parts. Every other axis is carried through unchanged.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L778", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L778", "path": "inference/model.py", "line": 778, "end_line": null}], "formula": "y = x^{(1)} \\,\\Vert\\, x^{(2)}, \\quad |t| = |w| + |s|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "804102503": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 804102503, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\lvert c \\rvert^{-1/2} x", "description": "This map multiplies every attention score by one over the square root of the latent width, which is 512 in the released model. A score is a sum over the 512 channels of a latent, so its spread grows with the square root of that width. The scale keeps the spread the same at any width, so the softmax does not put nearly all of its weight on one slot. The model drawn without the omitted mechanisms folds this factor into the query weight. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}], "formula": "y = x |c|_{512}^{-2^{-1}}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "278814693": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 278814693, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "z^{-1}", "description": "This map is the reciprocal of a normalising sum. The attention core divides each exponentiated score by the sum over its slots and the sink. The division is written as a product with this reciprocal. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = x^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2669548": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2669548, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1030759358": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1030759358, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "644792561": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 644792561, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Attention Core}", "description": "The attention core computes one head's softmax over the window slots, the selected entries and the learned sink. The two kinds of latent are laid end to end along one slot axis. The query scores against the latent at every slot. The scores are exponentiated. One copy is summed into the denominator together with the exponentiated sink logit. The reciprocal of that sum scales the exponentials contracted against the latents. Every score is multiplied by one over the square root of the latent width, which is 512 in the released model, before it is exponentiated. The scale keeps the spread of the scores the same at any width, so the softmax does not put nearly all of its weight on one slot. The sink logit is one learned number per head that lets the head give part of its weight to no slot. The sink logit is exponentiated with no scale.", "fill_color": "#C5BEDF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L639", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L639", "path": "inference/model.py", "line": 639, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L780", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L780", "path": "inference/model.py", "line": 780, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L392", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L392", "path": "inference/kernel.py", "line": 392, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L362-L364", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L362-L364", "path": "inference/kernel.py", "line": 362, "end_line": 364}, {"__type__": "CodeReference", "label": "inference/model.py L651", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L651", "path": "inference/model.py", "line": 651, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L365-L367", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L365-L367", "path": "inference/kernel.py", "line": 365, "end_line": 367}, {"__type__": "CodeReference", "label": "inference/kernel.py L382-L383", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L382-L383", "path": "inference/kernel.py", "line": 382, "end_line": 383}], "formula": null, "drawing": null}}, "777837084": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 777837084, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Pairs as Complex Numbers}", "description": "Reads every two adjacent channels as one complex number, with the first channel as the real part and the second as the imaginary part. A complex number is a point in a plane. Multiplying the point by a complex number of length one turns it about the origin, so the rotation of a pair is one product.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L397", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L397", "path": "inference/model.py", "line": 397, "end_line": null}], "formula": "p[i_{t}] = v[2 i_{t}] + \\mathrm{i}\\, v[2 i_{t} + 1]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1476696667": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1476696667, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "13468336": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 13468336, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Complex Numbers as Pairs}", "description": "Writes every complex number back as two adjacent channels, with the real part first and the imaginary part second. The operation undoes the reading of the channels as pairs.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404", "path": "inference/model.py", "line": 404, "end_line": null}], "formula": "y[2 i_{t}] = \\mathrm{Re}\\, p[i_{t}], \\qquad y[2 i_{t} + 1] = \\mathrm{Im}\\, p[i_{t}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "271908643": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 271908643, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2090611479": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2090611479, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Concatenation of the Channels}", "description": "Joins the channels left alone by the rotation and the rotated channels back into one vector. The run left alone comes first and the rotated run comes after it. The result has the channel axis the vector arrived on, so an operation that reads the vector reads the same axis whether or not the vector was rotated.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L404-L405", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L404-L405", "path": "inference/model.py", "line": 404, "end_line": 405}], "formula": "y[i_{\\bar{z}}] = u[i_{\\bar{z}}], \\qquad y[|\\bar{z}| + i_{z}] = r[i_{z}], \\qquad |c| = |\\bar{z}| + |z|", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "639391264": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 639391264, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "grp", "description": "Cuts one axis into consecutive groups of equal size, so a position of the result is a group and a place inside it. The attention cuts the heads into groups for the grouped output projection. The ratio-2 compressor cuts the tokens into entries of two. The pixel unshuffle of the image pathway cuts the rows of the patch grid into cell rows of three and its columns into cell columns of three.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L473-L474", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L473-L474", "path": "inference/model.py", "line": 473, "end_line": 474}, {"__type__": "CodeReference", "label": "inference/model.py L787", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L787", "path": "inference/model.py", "line": 787, "end_line": null}], "formula": "y[i_{g}, i_{j}] = x[|j|_{8}\\, i_{g} + i_{j}]", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1686887173": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1686887173, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "949498847": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 949498847, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reuse Attention}", "description": "A Reuse layer reads the entries and the selection from the tape, which holds what an earlier layer left for it. The layer therefore compresses no tokens and scores no entries. The entries arrive rotated and rounded. The layer computes its own query and window latents from its own hidden state. The query and the window latents are rotated by the positions of their tokens. The window latents are then rounded to the eight-bit FP8 format, as the released window cache holds them. The layer reads the latents of the selected entries and attends over the window and those entries under one softmax. The scores are multiplied by one over the square root of the latent width before the softmax. The result of the attention is rotated back by the position of its query before the output projection.", "fill_color": "#B8D8CE", "references": [{"__type__": "CodeReference", "label": "inference/model.py L1166-L1176", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1176", "path": "inference/model.py", "line": 1166, "end_line": 1176}, {"__type__": "CodeReference", "label": "inference/model.py L1180", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1180", "path": "inference/model.py", "line": 1180, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L700-L720", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L700-L720", "path": "inference/model.py", "line": 700, "end_line": 720}, {"__type__": "CodeReference", "label": "inference/model.py L762-L763", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L762-L763", "path": "inference/model.py", "line": 762, "end_line": 763}, {"__type__": "CodeReference", "label": "inference/model.py L765-L789", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L765-L789", "path": "inference/model.py", "line": 765, "end_line": 789}], "formula": null, "drawing": null}}, "1849789555": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1849789555, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "483985051": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 483985051, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1014701417": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1014701417, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "929600189": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 929600189, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "918339260": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 918339260, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sigma", "description": "The sigmoid is applied to the collapse vector of a hyper-connection, so the weight of each stream lies between zero and one. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1268719307": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1268719307, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1190960817": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1190960817, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "2\\sigma", "description": "Twice the sigmoid is applied to the output vector of a hyper-connection, so the weight of each stream lies between zero and two. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": null, "formula": "y = 2 \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1518295506": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1518295506, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x + \\varepsilon_{\\mathrm{hc}}", "description": "This map adds the epsilon of the hyper-connections, `hc_eps` in the released configuration, which is one millionth. The epsilon is added to the collapse vector after the sigmoid and to the combine matrix after the row softmax. Every value is therefore above zero. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L427", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L427", "path": "inference/kernel.py", "line": 427, "end_line": null}, {"__type__": "CodeReference", "label": "inference/kernel.py L443", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L443", "path": "inference/kernel.py", "line": 443, "end_line": null}], "formula": "y = \\varepsilon_{\\mathrm{hc}} + x", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1608689078": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1608689078, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "331677151": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 331677151, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1914135239": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1914135239, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "816346050": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 816346050, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x / \\tau", "description": "This map divides the logit of every expert by the temperature of the router, before the score function. A temperature of one leaves the logits unchanged. The released configuration sets none, so the released value is one. A larger temperature would bring the scores of the experts closer together. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L811", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L811", "path": "inference/model.py", "line": 811, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L67", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L67", "path": "inference/model.py", "line": 67, "end_line": null}], "formula": "y = x \\tau^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1682002804": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1682002804, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\sqrt{s^{+}}", "description": "The score function of the router is `sqrtsoftplus` in the released configuration. The softplus makes the score of an expert positive. The square root of the positive score is taken. The result is the gate of the expert. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L817", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L817", "path": "inference/model.py", "line": 817, "end_line": null}], "formula": "y = \\sqrt{\\ln(1 + e^{x})}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1660996569": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1660996569, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{TopK}", "description": "Keeps the largest scores along the axis consumed and hands each out at the position it came from. The result rides a sparse axis, labelled with the count kept and the axis selected over, which holds that many live positions. No index is therefore handed out beside the scores.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L822", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L822", "path": "inference/model.py", "line": 822, "end_line": null}], "formula": "y[i_{e}] = s[i_{e}] \\text{ where } s[i_{e}] \\text{ is among the } |k|_{6} \\text{ largest of } s", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "798728137": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 798728137, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Select}", "description": "Reads the payload at the positions kept by a selection and multiplies each value read by the value held there by the selection. The payload keeps its dense axis. The result rides the sparse axis of the selection.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L823", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L823", "path": "inference/model.py", "line": 823, "end_line": null}], "formula": "\\mathrm{Select}(w, p)[j] = w[j]\\, p[j] \\text{ at each live position } j", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2144658123": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2144658123, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\tfrac{3}{2}x", "description": "The six gates of a token sum to one after the normalisation. The route scale multiplies each gate by 1.5. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L826", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L826", "path": "inference/model.py", "line": 826, "end_line": null}], "formula": "y = 3 x 2^{-1}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "511710268": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 511710268, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "340596604": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 340596604, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "921535389": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 921535389, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1863083826": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1863083826, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "729541391": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 729541391, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1787220243": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1787220243, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1563989846": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1563989846, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1254897991": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1254897991, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "336061513": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 336061513, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2004048780": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2004048780, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1148025837": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1148025837, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "257179405": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 257179405, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\min(x, \\lambda)", "description": "This map limits the gate branch of an expert from above. A value above the SwiGLU limit, which is 10 in the released configuration, is replaced by the limit. Every other value is kept. The limit comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L847", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L847", "path": "inference/model.py", "line": 847, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = x - \\mathrm{ReLU}(x - \\lambda)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1862168246": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1862168246, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "x \\sigma(x)", "description": "The activation of an expert is SiLU. The gate projection passes through SiLU before it multiplies the up projection. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L848", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L848", "path": "inference/model.py", "line": 848, "end_line": null}], "formula": "y = x \\sigma(x)", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "286881087": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 286881087, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1267768102": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1267768102, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "description": "This map limits the up branch of an expert on both sides. A value above the SwiGLU limit is replaced by the limit. A value below minus the limit is replaced by minus the limit. Every other value is kept. The limit is 10 in the released configuration and comes from training, where it keeps the activations inside the range represented by the FP8 and FP4 number formats. An elementwise map. Every value x of the input is replaced by the value y the formula gives at it.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/model.py L846", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L846", "path": "inference/model.py", "line": 846, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L19", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L19", "path": "inference/config.json", "line": 19, "end_line": null}], "formula": "y = \\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "251959572": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 251959572, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2104822032": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2104822032, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast to MXFP8}", "description": "The released code runs this rounding kernel in front of every projection whose weight is FP8 or FP4. The channels of a row are cut into groups of 32. The largest magnitude of a group, kept at or above 10^-4, is divided by 448 and rounded up to a power of two, which is the UE8M0 scale of the group. Each channel is divided by the scale, kept between -448 and 448, and rounded to the nearest E4M3 number. The wire after the cast carries the E4M3 element of every channel and the scale of every group. The value denoted by the wire is the element multiplied by the scale. The GEMM reading the wire multiplies its FP32 accumulator by the scale of the operand and by the scale of the weight. The product is therefore of the values written above. A cast to E4M3 with no scale would saturate above 448 and flush to zero below 2^-9.", "fill_color": null, "references": [{"__type__": "CodeReference", "label": "inference/kernel.py L41-L96", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L41-L96", "path": "inference/kernel.py", "line": 41, "end_line": 96}, {"__type__": "CodeReference", "label": "inference/model.py L181-L206", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L181-L206", "path": "inference/model.py", "line": 181, "end_line": 206}, {"__type__": "CodeReference", "label": "inference/kernel.py L256-L269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L256-L269", "path": "inference/kernel.py", "line": 256, "end_line": 269}, {"__type__": "CodeReference", "label": "inference/kernel.py L544-L553", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/kernel.py#L544-L553", "path": "inference/kernel.py", "line": 544, "end_line": 553}], "formula": "\\begin{gathered}a[i_{g}] = \\max_{i_{y} \\in y} \\lvert x[32\\, i_{g} + i_{y}] \\rvert \\\\s[i_{g}] = 2^{\\lceil \\log_{2} (\\max(a[i_{g}],\\, 10^{-4}) / 448) \\rceil} \\\\\\hat{x}[32\\, i_{g} + i_{y}] = \\mathrm{E4M3}\\big(\\ulcorner x[32\\, i_{g} + i_{y}] / s[i_{g}] \\lrcorner_{-448}^{448}\\big)\\end{gathered}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "2036657155": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 2036657155, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1647750200": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1647750200, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1494622222": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1494622222, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1703232697": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1703232697, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1500849328": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1500849328, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Single-Pass mHC}", "description": "One sublayer runs inside the four-stream residual. The coefficients are predicted from the four streams. The streams are summed into one hidden state under the collapse vector predicted by the previous sublayer. The hidden state is normalised. The body runs on the result. The output vector times the body's result is added to the combine matrix applied to the four streams. The collapse vector predicted here goes to the next sublayer.", "fill_color": "#F1F4C1", "references": [{"__type__": "CodeReference", "label": "inference/model.py L957-L966", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L957-L966", "path": "inference/model.py", "line": 957, "end_line": 966}, {"__type__": "CodeReference", "label": "inference/model.py L981-L994", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L981-L994", "path": "inference/model.py", "line": 981, "end_line": 994}], "formula": null, "drawing": null}}, "724403640": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 724403640, "_name": null}, "repetition": {"__type__": "Integer", "_value": 3}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reuse Layer}", "description": "One of the three Reuse layers of a Reindex group: a Reuse attention sublayer reading the entries written by the decoder's Full layer and the selection written by the group's Reindex layer, and a mixture sublayer, each inside the four-stream residual.", "fill_color": "#F3F3F4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L27-L28", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L27-L28", "path": "inference/config.json", "line": 27, "end_line": 28}, {"__type__": "CodeReference", "label": "inference/model.py L1166-L1170", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1166-L1170", "path": "inference/model.py", "line": 1166, "end_line": 1170}], "formula": null, "drawing": null}}, "772823819": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 772823819, "_name": {"__type__": "DynamicName", "body": "l", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "repetition": {"__type__": "Integer", "_value": 4}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Reindex Group}", "description": "One of the four Reindex groups of the decoder, which are layers 24 to 39: a Reindex layer that scores inside the candidate pool and writes a fresh selection, followed by three Reuse layers that read it. Each iteration writes and reads the member of the selection named by the group counter l. The multimodal model writes the fourth group on its own, because DSpark reads the residual that enters each of its three Reuse layers. This model has no DSpark to read that residual.", "fill_color": "#EFEFF4", "references": [{"__type__": "CodeReference", "label": "inference/config.json L28", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L28", "path": "inference/config.json", "line": 28, "end_line": null}, {"__type__": "CodeReference", "label": "inference/config.json L52", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/config.json#L52", "path": "inference/config.json", "line": 52, "end_line": null}, {"__type__": "CodeReference", "label": "config.json L106", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/config.json#L106", "path": "config.json", "line": 106, "end_line": null}, {"__type__": "CodeReference", "label": "config.json L112", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/config.json#L112", "path": "config.json", "line": 112, "end_line": null}], "formula": null, "drawing": null}}, "1304171898": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1304171898, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1249667812": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1249667812, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in FP32 and writes it in BF16. The written format holds an element in 16 bits against the 32 of the operand's format, so every value is replaced by the nearest number the written format holds. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{FP32} \\to \\mathtt{BF16}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1708631622": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1708631622, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Cast}", "description": "The cast reads a value held in BF16 and writes it in FP32. The written format holds an element in 32 bits against the 16 of the operand's format, and holds every number that format holds, so no value changes. The wire on each side of the cast is labelled with its quantisation.", "fill_color": null, "references": null, "formula": "\\mathtt{BF16} \\to \\mathtt{FP32}", "drawing": {"__registered__": "enum", "type": "BlockDrawing", "name": "BODY_IN_PLACE"}}}, "1947590116": {"__type__": "FreeNumeric", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "FreeNumeric"}, "_id": 1947590116, "_name": {"__type__": "DynamicName", "body": "v", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": false, "absolute": true, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "129280", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}}, "2066137704": {"__type__": "RawAxis", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "RawAxis"}, "_id": 2066137704, "_name": {"__type__": "DynamicName", "body": "v", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": false, "overline": true, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUBSCRIPT"}}, "code_form": null, "exponent": {"__type__": "DynamicName", "body": "129280", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "_size": {"__ref__": 1947590116}}, "1781316040": {"__type__": "BlockTag", "uid": {"__type__": "UID", "_type": {"__registered__": "type", "repr": "BlockTag"}, "_id": 1781316040, "_name": null}, "repetition": {"__type__": "Integer", "_value": 1}, "aesthetics": {"__type__": "BlockAesthetics", "title": "\\text{Output Probabilities}", "description": "The final hidden state is normalised and projected onto the vocabulary. A softmax turns the projection into probabilities.", "fill_color": "#DBDFEF", "references": [{"__type__": "CodeReference", "label": "inference/model.py L1269", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1269", "path": "inference/model.py", "line": 1269, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L1012", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1012", "path": "inference/model.py", "line": 1012, "end_line": null}, {"__type__": "CodeReference", "label": "inference/model.py L1291", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/dba1be0a40aa45a94ad051997016db3960a90277/inference/model.py#L1291", "path": "inference/model.py", "line": 1291, "end_line": null}], "formula": null, "drawing": null}}}, "data": {"__type__": "Composed", "content": [{"__type__": "Rearrangement", "mapping": [0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "CovariantView", "name": {"__type__": "DynamicName", "body": "A^{0}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "reindexing": {"__type__": "StrideMorphism", "_dom": [], "_cod_stride_shift": [[{"__ref__": 1330421004}, [], {"__type__": "Integer", "_value": 0}]], "name": {"__type__": "DynamicName", "body": "A^{0}", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": [{"__ref__": 777392202}]}}, "block_tag": {"__ref__": 621350211}}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}, {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Embedding", "name": {"__type__": "DynamicName", "body": "E", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 2119517555}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, "block_tag": {"__ref__": 917611616}}, {"__type__": "ParaWrap", "body": {"__type__": "Broadcasted", "operator": {"__type__": "ParaBlockOperator", "name": {"__type__": "DynamicName", "body": "In", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "ParaWrap", "body": {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}]}, "grabs": [null], "drops": [{"__ref__": 1636457132}]}, "block_tag": {"__ref__": 1384614369}}, "grabs": [], "drops": [{"__type__": "Drop", "tape": {"__ref__": 1636457132}, "size": {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}}]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, "grabs": [null], "drops": [{"__ref__": 1636457132}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "rep", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}]}]}, {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Rearrangement", "mapping": [0, 1, 1], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Coef", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 242910120}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": false, "bias": false, "epsilon": {"__ref__": 1391415111}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0, 0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "H", "subscript": {"__type__": "DynamicName", "body": "0", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": true}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\sigma", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Sigmoid", "argument": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 1275413849}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x + \\varepsilon_{\\mathrm{hc}}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Addition", "content": [{"__ref__": 1711934402}, {"__type__": "FreeInput"}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 1349575359}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "H", "subscript": {"__type__": "DynamicName", "body": "1", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": true}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "2\\sigma", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": 2}, {"__type__": "Sigmoid", "argument": {"__type__": "FreeInput"}}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 109908815}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "H", "subscript": {"__type__": "DynamicName", "body": "2", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": true}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "e^{x}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Power", "base": {"__type__": "Constant", "symbol": {"__registered__": "enum", "type": "ConstantSymbol", "name": "EULER"}}, "exponent": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__type__": "Integer", "_value": 0}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x + \\varepsilon_{\\mathrm{hc}}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Addition", "content": [{"__ref__": 1711934402}, {"__type__": "FreeInput"}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 2030479332}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__ref__": 1711934402}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}]}, "block_tag": {"__ref__": 579469646}}, {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__ref__": 1711934402}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__ref__": 1711934402}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}]}, "block_tag": {"__ref__": 321235664}}]}]}]}, "block_tag": {"__ref__": 308478123}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}]}, {"__type__": "Rearrangement", "mapping": [2, 0, 1, 3, 4], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 1330421004}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 999722612}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 1330421004}]}]}]}, {"__type__": "Rearrangement", "mapping": [0, 1, 2, 1, 3], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 1330421004}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0], [0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [], "_dom": [{"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 2095719082}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": true, "bias": false, "epsilon": {"__ref__": 758490154}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "SWA", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1021115887}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "\\mathrm{sink}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 706534913}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{Qa}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1128010231}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": true, "bias": false, "epsilon": {"__ref__": 758490154}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1128010231}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1128010231}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1128010231}]}], "backup_degree": null}, "block_tag": {"__ref__": 1351821723}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1128010231}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1128010231}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{Qb}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1128010231}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 706534913}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "tr", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [1, 0, 2], "_dom": [{"__ref__": 706534913}, {"__ref__": 777392202}, {"__ref__": 1056277277}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Rot", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "DeconcatenateAxes", "name": {"__type__": "DynamicName", "body": "\\Vert^{-1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_reindexings": [{"__type__": "StrideMorphism", "_dom": [{"__ref__": 182489280}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": null}, {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1821531651}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Addition", "content": [{"__ref__": 296734870}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -2}, {"__ref__": 1181195560}]}]}]], "name": null}]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 182489280}]}]}, {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "PairsAsComplex", "name": {"__type__": "DynamicName", "body": "PairsAsComplex", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 29130293}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "target": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "backup_degree": null}, "block_tag": {"__ref__": 1000427244}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Rotary", "name": {"__type__": "DynamicName", "body": "\\mathrm{RoPE}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "base": {"__ref__": 1477143043}, "position_stride": {"__type__": "Integer", "_value": 1}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}, {"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Decomplex", "name": {"__type__": "DynamicName", "body": "Decomplex", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 1381921121}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "backup_degree": null}, "block_tag": {"__ref__": 356046016}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "ConcatenateAxes", "name": {"__type__": "DynamicName", "body": "\\Vert", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_reindexings": [{"__type__": "StrideMorphism", "_dom": [{"__ref__": 182489280}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": null}, {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1821531651}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Addition", "content": [{"__ref__": 296734870}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -2}, {"__ref__": 1181195560}]}]}]], "name": null}]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 852155779}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}, {"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 923799071}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 777392202}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 777392202}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 706534913}]}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{KV}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": true, "bias": false, "epsilon": {"__ref__": 758490154}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Rot", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "DeconcatenateAxes", "name": {"__type__": "DynamicName", "body": "\\Vert^{-1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_reindexings": [{"__type__": "StrideMorphism", "_dom": [{"__ref__": 182489280}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": null}, {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1821531651}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Addition", "content": [{"__ref__": 296734870}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -2}, {"__ref__": 1181195560}]}]}]], "name": null}]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 182489280}]}]}, {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "PairsAsComplex", "name": {"__type__": "DynamicName", "body": "PairsAsComplex", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 1811534717}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "target": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "backup_degree": null}, "block_tag": {"__ref__": 1388372297}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Rotary", "name": {"__type__": "DynamicName", "body": "\\mathrm{RoPE}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "base": {"__ref__": 1477143043}, "position_stride": {"__type__": "Integer", "_value": 1}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}, {"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Decomplex", "name": {"__type__": "DynamicName", "body": "Decomplex", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 30666566}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "backup_degree": null}, "block_tag": {"__ref__": 1559875282}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "ConcatenateAxes", "name": {"__type__": "DynamicName", "body": "\\Vert", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_reindexings": [{"__type__": "StrideMorphism", "_dom": [{"__ref__": 182489280}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": null}, {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1821531651}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Addition", "content": [{"__ref__": 296734870}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -2}, {"__ref__": 1181195560}]}]}]], "name": null}]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 1375795238}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}, {"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 923799071}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "FP8w", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "qgrp", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Block", "body": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__ref__": 903385593}, {"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": {"__type__": "DynamicName", "body": "qgrp", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "block_tag": {"__ref__": 72447950}}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "backup_degree": null}, "block_tag": {"__ref__": 85460599}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\lvert x \\rvert", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "AbsoluteValue", "argument": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "backup_degree": null}, "block_tag": {"__ref__": 693463748}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Maximum", "name": {"__type__": "DynamicName", "body": "\\max", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1981822304}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\max(x, 10^{-4})", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "LargerOf", "first": {"__type__": "FreeInput"}, "second": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 10}, "exponent": {"__type__": "Integer", "_value": -4}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}]}], "backup_degree": null}, "block_tag": {"__ref__": 1002981784}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x / 448", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Multiplication", "content": [{"__type__": "FreeInput"}, {"__type__": "Power", "base": {"__type__": "Integer", "_value": 448}, "exponent": {"__type__": "Integer", "_value": -1}}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}]}], "backup_degree": null}, "block_tag": {"__ref__": 899603404}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\log_{2} x", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Logarithm", "base": {"__type__": "Integer", "_value": 2}, "argument": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}]}], "backup_degree": null}, "block_tag": {"__ref__": 2069837421}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "GenericOperator", "name": {"__type__": "DynamicName", "body": "\\lceil x \\rceil", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}]}], "backup_degree": null}, "block_tag": {"__ref__": 1652884106}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "2^{x}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}]}], "backup_degree": null}]}]}, {"__type__": "Rearrangement", "mapping": [0, 1, 1], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x^{-1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Power", "base": {"__type__": "FreeInput"}, "exponent": {"__type__": "Integer", "_value": -1}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}]}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\ulcorner x \\lrcorner_{-448}^{448}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Clamp", "argument": {"__type__": "FreeInput"}, "lower": {"__type__": "Integer", "_value": -448}, "upper": {"__type__": "Integer", "_value": 448}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "backup_degree": null}, "block_tag": {"__ref__": 1058467236}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "backup_degree": null}, "block_tag": {"__ref__": 1031685272}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "backup_degree": null}, "block_tag": {"__ref__": 2094582236}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}]}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "CovariantView", "name": {"__type__": "DynamicName", "body": "qgrp", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "reindexing": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 835835122}, {"__ref__": 1981822304}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__ref__": 903385593}, {"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": {"__type__": "DynamicName", "body": "qgrp", "subscript": null, "settings": null, "code_form": null, "exponent": null}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, "block_tag": {"__ref__": 1257516800}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 835835122}, {"__ref__": 1981822304}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1056277277}]}], "backup_degree": null}, "block_tag": {"__ref__": 1446818017}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 207717483}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "win", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Block", "body": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 777392202}, {"__ref__": 1922576489}], "_cod_stride_shift": [[{"__ref__": 777392202}, [{"__type__": "Integer", "_value": 1}, {"__type__": "Integer", "_value": -1}], {"__type__": "Integer", "_value": 0}]], "name": {"__type__": "DynamicName", "body": "win", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "block_tag": {"__ref__": 95011573}}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1056277277}]}]}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Core", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Rearrangement", "mapping": [0, 1, 2, 2], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": []}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}, {"__ref__": 1056277277}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": []}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0], [0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}, {"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\lvert c \\rvert^{-1/2} x", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Multiplication", "content": [{"__type__": "FreeInput"}, {"__type__": "Power", "base": {"__ref__": 296734870}, "exponent": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": -1}}]}}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}], "backup_degree": null}, "block_tag": {"__ref__": 86086985}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "e^{x}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Power", "base": {"__type__": "Constant", "symbol": {"__registered__": "enum", "type": "ConstantSymbol", "name": "EULER"}}, "exponent": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}], "backup_degree": null}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}, {"__ref__": 1056277277}]}]}]}, {"__type__": "Rearrangement", "mapping": [0, 1, 1, 2], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": []}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}, {"__ref__": 1056277277}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "e^{x}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Power", "base": {"__type__": "Constant", "symbol": {"__registered__": "enum", "type": "ConstantSymbol", "name": "EULER"}}, "exponent": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": []}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": []}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1922576489}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "AdditionOp", "name": {"__type__": "DynamicName", "body": "+", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_of_fma": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": []}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "z^{-1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Power", "base": {"__type__": "FreeInput"}, "exponent": {"__type__": "Integer", "_value": -1}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 1845635261}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}], "backup_degree": null}, "block_tag": {"__ref__": 1043465010}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}, {"__ref__": 1056277277}]}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0], [0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1922576489}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1922576489}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}, {"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}, {"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "backup_degree": null}, "block_tag": {"__ref__": 1392885062}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 1278767244}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 777392202}, {"__ref__": 1056277277}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1922576489}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 777392202}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 706534913}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 706534913}]}, {"__type__": "Rearrangement", "mapping": [], "_dom": [{"__ref__": 706534913}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Rot^{-1}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "DeconcatenateAxes", "name": {"__type__": "DynamicName", "body": "\\Vert^{-1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_reindexings": [{"__type__": "StrideMorphism", "_dom": [{"__ref__": 182489280}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": null}, {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1821531651}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Addition", "content": [{"__ref__": 296734870}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -2}, {"__ref__": 1181195560}]}]}]], "name": null}]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 182489280}]}]}, {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "PairsAsComplex", "name": {"__type__": "DynamicName", "body": "PairsAsComplex", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 1703485446}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "target": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "backup_degree": null}, "block_tag": {"__ref__": 1601504201}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Rotary", "name": {"__type__": "DynamicName", "body": "\\mathrm{RoPE}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "base": {"__ref__": 1477143043}, "position_stride": {"__type__": "Integer", "_value": 1}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}, {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\overline{x}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Conjugate", "argument": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}, {"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Decomplex", "name": {"__type__": "DynamicName", "body": "Decomplex", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 1907365617}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Complex", "base": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 35354379}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "backup_degree": null}, "block_tag": {"__ref__": 617321454}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "ConcatenateAxes", "name": {"__type__": "DynamicName", "body": "\\Vert", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_reindexings": [{"__type__": "StrideMorphism", "_dom": [{"__ref__": 182489280}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": null}, {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1821531651}], "_cod_stride_shift": [[{"__ref__": 1056277277}, [{"__type__": "Integer", "_value": 1}], {"__type__": "Addition", "content": [{"__ref__": 296734870}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -2}, {"__ref__": 1181195560}]}]}]], "name": null}]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 1215894946}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 182489280}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1821531651}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}, {"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 376895794}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 777392202}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 777392202}, {"__ref__": 1056277277}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 706534913}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "tr", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [1, 0, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 706534913}, {"__ref__": 1056277277}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Out", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "grp", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Block", "body": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 13639104}, {"__ref__": 182848503}], "_cod_stride_shift": [[{"__ref__": 706534913}, [{"__ref__": 141018594}, {"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": {"__type__": "DynamicName", "body": "grp", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "block_tag": {"__ref__": 792264637}}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1056277277}]}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{Oa}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 182848503}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 13639104}, {"__ref__": 55955818}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 13639104}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "diag", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 0, 1], "_dom": [{"__ref__": 13639104}, {"__ref__": 55955818}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 13639104}, {"__ref__": 55955818}]}], "backup_degree": null}, "block_tag": {"__ref__": 1551896111}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 13639104}, {"__ref__": 55955818}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 13639104}, {"__ref__": 55955818}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{Ob}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 13639104}, {"__ref__": 55955818}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 1309719187}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 706534913}, {"__ref__": 1056277277}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}]}, "block_tag": {"__ref__": 517862890}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 407386606}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0], [0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "AdditionOp", "name": {"__type__": "DynamicName", "body": "+", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_of_fma": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}, {"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 807017432}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}]}, "block_tag": {"__ref__": 937953296}}, {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Rearrangement", "mapping": [0, 1, 1], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Coef", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 36706552}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": false, "bias": false, "epsilon": {"__ref__": 1391415111}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0, 0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "H", "subscript": {"__type__": "DynamicName", "body": "0", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": true}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\sigma", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Sigmoid", "argument": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 411460338}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x + \\varepsilon_{\\mathrm{hc}}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Addition", "content": [{"__ref__": 1711934402}, {"__type__": "FreeInput"}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 1043665725}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "H", "subscript": {"__type__": "DynamicName", "body": "1", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": true}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "2\\sigma", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": 2}, {"__type__": "Sigmoid", "argument": {"__type__": "FreeInput"}}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 504897397}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "H", "subscript": {"__type__": "DynamicName", "body": "2", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": true}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "e^{x}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Power", "base": {"__type__": "Constant", "symbol": {"__registered__": "enum", "type": "ConstantSymbol", "name": "EULER"}}, "exponent": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__type__": "Integer", "_value": 0}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x + \\varepsilon_{\\mathrm{hc}}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Addition", "content": [{"__ref__": 1711934402}, {"__type__": "FreeInput"}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 1875313949}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__ref__": 1711934402}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}]}, "block_tag": {"__ref__": 579469646}}, {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__ref__": 1711934402}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__ref__": 1711934402}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}]}], "backup_degree": null}]}, "block_tag": {"__ref__": 321235664}}]}]}]}, "block_tag": {"__ref__": 308478123}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}]}, {"__type__": "Rearrangement", "mapping": [2, 0, 1, 3, 4], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 1330421004}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 836795547}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 1330421004}]}]}]}, {"__type__": "Rearrangement", "mapping": [0, 1, 2, 1, 3], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 1330421004}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0], [0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [], "_dom": [{"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1086684913}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": true, "bias": false, "epsilon": {"__ref__": 758490154}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "MoE", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Rearrangement", "mapping": [0, 0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Gate", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 85202227}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{R}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x / \\tau", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Multiplication", "content": [{"__type__": "FreeInput"}, {"__type__": "Power", "base": {"__ref__": 1627256985}, "exponent": {"__type__": "Integer", "_value": -1}}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 401190795}]}], "backup_degree": null}, "block_tag": {"__ref__": 182144126}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\sqrt{s^{+}}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Power", "base": {"__type__": "Logarithm", "base": {"__type__": "Constant", "symbol": {"__registered__": "enum", "type": "ConstantSymbol", "name": "EULER"}}, "argument": {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Power", "base": {"__type__": "Constant", "symbol": {"__registered__": "enum", "type": "ConstantSymbol", "name": "EULER"}}, "exponent": {"__type__": "FreeInput"}}]}}, "exponent": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": -1}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 401190795}]}], "backup_degree": null}, "block_tag": {"__ref__": 636446273}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "\\mathrm{bias}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}]}, {"__type__": "Broadcasted", "operator": {"__type__": "AdditionOp", "name": {"__type__": "DynamicName", "body": "+", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_of_fma": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 401190795}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 401190795}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TopK", "name": {"__type__": "DynamicName", "body": "TopK", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "k": {"__ref__": 1323411805}, "form": {"__registered__": "enum", "type": "SelectionForm", "name": "WEIGHTS"}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, "block_tag": {"__ref__": 296967158}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\mathbbm{1}_{x > 0}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "IsPositive", "argument": {"__type__": "FreeInput"}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1137802141}]}], "backup_degree": null}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Select", "name": {"__type__": "DynamicName", "body": "Select", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}, {"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, "block_tag": {"__ref__": 1615673735}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 401190795}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}, {"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "L1Norm", "name": {"__type__": "DynamicName", "body": "L^{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "contracted": false, "epsilon": {"__ref__": 1214936271}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\tfrac{3}{2}x", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": 3}, {"__type__": "FreeInput"}, {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": -1}}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1137802141}]}], "backup_degree": null}, "block_tag": {"__ref__": 1182676660}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 1189046433}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Composed", "content": [{"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 2129965811}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{G}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 432449881}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\min(x, \\lambda)", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Addition", "content": [{"__type__": "FreeInput"}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__type__": "RectifiedLinear", "argument": {"__type__": "Addition", "content": [{"__type__": "FreeInput"}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 96553797}]}]}}]}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 671146010}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x \\sigma(x)", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Multiplication", "content": [{"__type__": "FreeInput"}, {"__type__": "Sigmoid", "argument": {"__type__": "FreeInput"}}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 2070052423}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{U}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 916785349}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Clamp", "argument": {"__type__": "FreeInput"}, "lower": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 96553797}]}, "upper": {"__ref__": 96553797}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 164042319}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1137802141}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1137802141}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 1471042596}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 511925873}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{D}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1137802141}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1137802141}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "diag", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 975060836}]}], "backup_degree": null}]}, "block_tag": {"__ref__": 1083357423}}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1137802141}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1379230893}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0], [0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1137802141}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": [{"__ref__": 975060836}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1003648250}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{Gs}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 131843983}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\min(x, \\lambda)", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Addition", "content": [{"__type__": "FreeInput"}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__type__": "RectifiedLinear", "argument": {"__type__": "Addition", "content": [{"__type__": "FreeInput"}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 96553797}]}]}}]}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 1683887822}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "x \\sigma(x)", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Multiplication", "content": [{"__type__": "FreeInput"}, {"__type__": "Sigmoid", "argument": {"__type__": "FreeInput"}}]}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 1409242776}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{Us}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 1511701545}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\ulcorner x \\lrcorner_{-\\lambda}^{\\lambda}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Clamp", "argument": {"__type__": "FreeInput"}, "lower": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 96553797}]}, "upper": {"__ref__": 96553797}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 2123592162}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 1732281219}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 435880113}]}], "backup_degree": null}, "block_tag": {"__ref__": 461059629}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 435880113}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{Ds}", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 435880113}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 55829965}}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 811098828}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "AdditionOp", "name": {"__type__": "DynamicName", "body": "+", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_of_fma": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1418510891}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 1758278491}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1582799004}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0], [0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}]}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "AdditionOp", "name": {"__type__": "DynamicName", "body": "+", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_of_fma": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}, {"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1556924115}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Reals"}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}]}]}, "block_tag": {"__ref__": 821503956}}]}, "block_tag": {"__ref__": 1789063455}}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "ParaWrap", "body": {"__type__": "Broadcasted", "operator": {"__type__": "ParaBlockOperator", "name": {"__type__": "DynamicName", "body": "Eng", "subscript": {"__type__": "DynamicName", "body": "{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Rearrangement", "mapping": [0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1779995924}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "Composed", "content": [{"__type__": "Block", "body": {"__type__": "ParaWrap", "body": {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Embedding", "name": {"__type__": "DynamicName", "body": "E", "subscript": {"__type__": "DynamicName", "body": "\\mathrm{cmp}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, "block_tag": {"__ref__": 5265176}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, "grabs": [{"__ref__": 1636457132}], "drops": [null]}, "block_tag": {"__ref__": 188737552}}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "ngram", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Block", "body": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 777392202}, {"__ref__": 943552329}], "_cod_stride_shift": [[{"__ref__": 777392202}, [{"__type__": "Integer", "_value": 1}, {"__type__": "Integer", "_value": -1}], {"__type__": "Integer", "_value": 0}]], "name": {"__type__": "DynamicName", "body": "ngram", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "block_tag": {"__ref__": 1906749507}}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Hash", "subscript": {"__type__": "DynamicName", "body": "{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 943552329}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "FixedArray", "name": {"__type__": "DynamicName", "body": "p", "subscript": {"__type__": "DynamicName", "body": "{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 912013943}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}, "block_tag": {"__ref__": 70620287}}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 912013943}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}]}, {"__type__": "Rearrangement", "mapping": [0, 1, 1], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 943552329}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 912013943}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 943552329}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "FixedArray", "name": {"__type__": "DynamicName", "body": "\\alpha", "subscript": {"__type__": "DynamicName", "body": "{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Multiplication", "content": [{"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}, {"__type__": "Power", "base": {"__ref__": 142403013}, "exponent": {"__type__": "Integer", "_value": -1}}]}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 943552329}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}, "block_tag": {"__ref__": 2031763661}}}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Multiplication", "content": [{"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}, {"__type__": "Power", "base": {"__ref__": 142403013}, "exponent": {"__type__": "Integer", "_value": -1}}]}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 943552329}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Multiplication", "content": [{"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}, {"__type__": "Power", "base": {"__ref__": 142403013}, "exponent": {"__type__": "Integer", "_value": -1}}]}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 943552329}]}, {"__type__": "Rearrangement", "mapping": [1], "_dom": [{"__ref__": 777392202}, {"__ref__": 943552329}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "prefix", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}, {"__type__": "Block", "body": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1261606321}, {"__ref__": 1379823832}], "_cod_stride_shift": [[{"__ref__": 943552329}, [{"__type__": "Integer", "_value": 1}, {"__type__": "Integer", "_value": 1}], {"__type__": "Addition", "content": [{"__type__": "Integer", "_value": 1}, {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 1178248995}]}]}]], "name": {"__type__": "DynamicName", "body": "prefix", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "block_tag": {"__ref__": 1943498271}}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BitwiseXor", "name": {"__type__": "DynamicName", "body": "\\oplus", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1379823832}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1261606321}]}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "pair", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Block", "body": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1261606321}, {"__ref__": 1345757761}], "_cod_stride_shift": [[{"__ref__": 912013943}, [{"__ref__": 1396444206}, {"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": {"__type__": "DynamicName", "body": "pair", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "block_tag": {"__ref__": 270461316}}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Modulo", "name": {"__type__": "DynamicName", "body": "\\bmod", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": 63}}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}, {"__type__": "Rearrangement", "mapping": [1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}], "backup_degree": null}]}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "before", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Block", "body": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 912013943}, {"__ref__": 334943757}], "_cod_stride_shift": [[{"__ref__": 912013943}, [{"__type__": "Integer", "_value": 1}, {"__type__": "Integer", "_value": 1}], {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__ref__": 1178248995}, {"__ref__": 1396444206}]}]], "name": {"__type__": "DynamicName", "body": "before", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "block_tag": {"__ref__": 1483777515}}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 334943757}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 912013943}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "View", "name": {"__type__": "DynamicName", "body": "pair", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": null}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Block", "body": {"__type__": "StrideMorphism", "_dom": [{"__ref__": 1261606321}, {"__ref__": 1345757761}], "_cod_stride_shift": [[{"__ref__": 912013943}, [{"__ref__": 1396444206}, {"__type__": "Integer", "_value": 1}], {"__type__": "Integer", "_value": 0}]], "name": {"__type__": "DynamicName", "body": "pair", "subscript": null, "settings": null, "code_form": null, "exponent": null}}, "block_tag": {"__ref__": 1468796888}}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "AdditionOp", "name": {"__type__": "DynamicName", "body": "+", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_of_fma": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 1390890971}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Addition", "content": [{"__ref__": 336459836}, {"__ref__": 1390890971}]}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}, {"__type__": "Rearrangement", "mapping": [1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Cast", "name": {"__type__": "DynamicName", "body": "\\mathrm{cast}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "cast"}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Addition", "content": [{"__ref__": 336459836}, {"__ref__": 1390890971}]}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}], "backup_degree": null}, "block_tag": {"__ref__": 749433221}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__type__": "Addition", "content": [{"__ref__": 336459836}, {"__ref__": 1390890971}]}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 1789295170}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 142403013}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 943552329}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Embedding", "name": {"__type__": "DynamicName", "body": "E", "subscript": {"__type__": "DynamicName", "body": "\\mathrm{ng}", "subscript": {"__type__": "DynamicName", "body": "{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 745749420}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}], "backup_degree": null}, "block_tag": {"__ref__": 43558003}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 336459836}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}, {"__ref__": 745749420}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Rearrangement", "mapping": [0, 0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1261606321}, {"__ref__": 1345757761}, {"__ref__": 745749420}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{K}", "subscript": {"__type__": "DynamicName", "body": "{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1261606321}, {"__ref__": 1345757761}, {"__ref__": 745749420}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "W^{V}", "subscript": {"__type__": "DynamicName", "body": "{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 8}, "vector": {"__type__": "Integer", "_value": 4}, "form": {"__registered__": "enum", "type": "Encoding", "name": "E4M3"}, "scale": {"__type__": "BlockScale", "form": {"__registered__": "enum", "type": "Encoding", "name": "UE8M0"}, "channels": {"__type__": "Integer", "_value": 32}, "rows": {"__type__": "Integer", "_value": 1}}}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 1261606321}, {"__ref__": 1345757761}, {"__ref__": 745749420}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__ref__": 777392202}]}], "backup_degree": null}]}]}, "block_tag": {"__ref__": 978840573}}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "ProductOfMorphisms", "content": [{"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1601218126}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": false, "bias": false, "epsilon": {"__ref__": 1391415111}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Linear", "name": {"__type__": "DynamicName", "body": "\\mathrm{w}", "subscript": {"__type__": "DynamicName", "body": "{1}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "bias": false}, "input_weaves": [], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [], "backup_degree": {"__type__": "ProdObject", "content": []}}, {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1037965390}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": false, "bias": false, "epsilon": {"__ref__": 1391415111}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[0], [0], [0]]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Rearrangement", "mapping": [1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "Arithmetic", "name": {"__type__": "DynamicName", "body": "\\mathrm{gate}", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "operator": "arithmetic", "formula": {"__type__": "Sigmoid", "argument": {"__type__": "Multiplication", "content": [{"__type__": "Sign", "argument": {"__type__": "Multiplication", "content": [{"__type__": "FreeInput"}, {"__type__": "Power", "base": {"__ref__": 2073802315}, "exponent": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": -1}}]}}]}}, {"__type__": "SquareRoot", "argument": {"__type__": "LargerOf", "first": {"__type__": "AbsoluteValue", "argument": {"__type__": "Multiplication", "content": [{"__type__": "FreeInput"}, {"__type__": "Power", "base": {"__ref__": 2073802315}, "exponent": {"__type__": "Multiplication", "content": [{"__type__": "Integer", "_value": -1}, {"__type__": "Power", "base": {"__type__": "Integer", "_value": 2}, "exponent": {"__type__": "Integer", "_value": -1}}]}}]}}, "second": {"__ref__": 1455641400}}}]}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}], "backup_degree": null}, "block_tag": {"__ref__": 1876072715}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 1172950056}}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1817300803}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, {"__type__": "Broadcasted", "operator": {"__type__": "Einops", "name": {"__type__": "DynamicName", "body": "einops", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "signature": [[], []]}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}, {"__type__": "Rearrangement", "mapping": [0, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "AdditionOp", "name": {"__type__": "DynamicName", "body": "+", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "part_of_fma": false}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}, {"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1, 2], "_dom": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1509955038}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}]}, "block_tag": {"__ref__": 77275471}}, "grabs": [{"__type__": "Grab", "tape": {"__ref__": 1636457132}, "size": {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}}], "drops": []}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Natural", "max_value": {"__ref__": 277713878}}, "size": {"__type__": "Integer", "_value": 64}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "INT64"}, "scale": null}, "_shape": [{"__ref__": 777392202}]}, {"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}, {"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, "grabs": [{"__ref__": 1636457132}, null], "drops": [null]}]}, {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Rearrangement", "mapping": [0, 1, 1], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}, {"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "ProductOfMorphisms", "content": [{"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}]}]}, {"__type__": "Rearrangement", "mapping": [0], "_dom": [{"__type__": "Array", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 777392202}, {"__ref__": 1330421004}, {"__ref__": 975060836}]}]}, {"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": {"__type__": "DynamicName", "body": "Coef", "subscript": null, "settings": {"__type__": "DynamicNameSettings", "bold": true, "overline": false, "absolute": false, "typewriter": false, "exponent_placement": {"__registered__": "enum", "type": "ExponentPlacement", "name": "SUPERSCRIPT"}}, "code_form": null, "exponent": null}, "block": {"__type__": "Block", "body": {"__type__": "Composed", "content": [{"__type__": "Broadcasted", "operator": {"__type__": "BlockOperator", "name": null, "block": {"__type__": "Block", "body": {"__type__": "Broadcasted", "operator": {"__type__": "TypeConvert", "name": null, "source": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "target": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}, {"__registered__": "enum", "type": "WeaveMode", "name": "TILED"}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [0, 1], "_dom": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "backup_degree": null}, "block_tag": {"__ref__": 1317094583}}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 16}, "vector": {"__type__": "Integer", "_value": 2}, "form": {"__registered__": "enum", "type": "Encoding", "name": "BF16"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "reindexings": [{"__type__": "Rearrangement", "mapping": [], "_dom": []}], "backup_degree": null}, {"__type__": "Broadcasted", "operator": {"__type__": "Normalize", "name": {"__type__": "DynamicName", "body": "RMSNorm", "subscript": null, "settings": null, "code_form": null, "exponent": null}, "gain": false, "bias": false, "epsilon": {"__ref__": 1391415111}}, "input_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__ref__": 1330421004}, {"__ref__": 975060836}]}], "output_weaves": [{"__type__": "Weave", "datatype": {"__type__": "Quantified", "wraps": {"__type__": "Reals"}, "size": {"__type__": "Integer", "_value": 32}, "vector": {"__type__": "Integer", "_value": 1}, "form": {"__registered__": "enum", "type": "Encoding", "name": "FP32"}, "scale": null}, "_shape": [{"__r

tsncd