{ "source": "https://sebastianraschka.com/llm-architecture-gallery/", "syncedAt": "2026-11-04T16:15:11.462Z", "count": 118, "models": [ { "slug": "mistral-large-4", "order": 117, "name": "Mistral Large 4 (1T)", "baseTitle": "Mistral Large 4", "company": "Mistral AI", "country": "France", "flag": "🇫🇷", "date": "2026-10-29", "decoderType": "Sparse MoE", "attention": "MLA with modified DeepSeek Sparse Attention, index reuse, and KV sharing", "layerMix": "72 MLA/DSA", "scale": "1T total, 52B active (5.2% active)", "totalParams": 1000000000000, "activeParams": 52000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "Mistral Ecosystem Open-Weight License", "kvCacheText": "80.75 KiB", "kvCacheKiB": 80.75, "kvCacheLevel": "Moderate", "summary": "Mistral's trillion-parameter multimodal MoE combines MLA with sparse token selection and cross-layer KV sharing.", "keyDetail": "Selects the top 2,048 tokens, reuses sparse indices across layers, and alternates three LeRoPE layers with one NoPE layer.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/mistral-large-4.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/mistral-large-4.webp", "alt": "Figure 1: Mistral Large 4", "width": 960, "height": 833 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/mistralai/Mistral-Large-4-1T-A52B/blob/main/config.json", "title": "mistralai/Mistral-Large-4-1T-A52B" }, { "label": "License", "href": "https://huggingface.co/mistralai/Mistral-Large-4-1T-A52B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://huggingface.co/mistralai/Mistral-Large-4-1T-A52B/blob/main/assets/technical_report.pdf", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "indexshare", "label": "IndexShare", "name": "IndexShare", "url": "https://llm-atlas-7or.pages.dev/concepts/indexshare/" }, { "slug": "kv-sharing", "label": "KV Sharing", "name": "Cross-Layer KV Sharing", "url": "https://llm-atlas-7or.pages.dev/concepts/kv-sharing/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 26, "profile": { "Knowledge Work": 40.6, "Coding": 18.9, "Science": 13, "General": 31.7 }, "url": "https://artificialanalysis.ai/models/mistral-large-4" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-mistral-large-4", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/mistral-large-4/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/mistral-large-4.md" }, { "slug": "lfm3-1-7b", "order": 116, "name": "LFM3 (1.7B)", "baseTitle": "LFM3 1.7B", "company": "Liquid AI", "country": "United States", "flag": "🇺🇸", "date": "2026-10-27", "decoderType": "Dense hybrid", "attention": "ShortConv plus gated GQA with QK-Norm and sliding-window/global attention", "layerMix": "13 ShortConv + 5 sliding-window + 6 global GQA", "scale": "1.73B parameters", "totalParams": 1730000000, "activeParams": 1730000000, "context": "262,144", "contextTokens": 262144, "license": "LFM Open License v1.0", "kvCacheText": "22 KiB", "kvCacheKiB": 22, "kvCacheLevel": "Very low", "summary": "Liquid AI's 1.73B dense hybrid combines short convolutions with sliding-window and global GQA in a 24-layer stack.", "keyDetail": "Uses kernel-7 ShortConv, a 256-token attention window, per-head sigmoid gates, and sandwich RMSNorm. Global attention uses 50% partial RoPE.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/lfm3-1-7b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/lfm3-1-7b.webp", "alt": "Figure 1: LFM3 1.7B", "width": 833, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/LiquidAI/LFM3-1.7B/blob/main/config.json", "title": "LiquidAI/LFM3-1.7B" }, { "label": "License", "href": "https://huggingface.co/LiquidAI/LFM3-1.7B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://huggingface.co/blog/LiquidAI/lfm3", "title": null } ], "concepts": [ { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-lfm3-1-7b", "family": "hybrid", "url": "https://llm-atlas-7or.pages.dev/models/lfm3-1-7b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/lfm3-1-7b.md" }, { "slug": "minimax-m3-1-flash-428b", "order": 115, "name": "MiniMax M3.1-Flash (428B)", "baseTitle": "MiniMax M3.1-Flash 428B", "company": "MiniMax", "country": "China", "flag": "🇨🇳", "date": "2026-10-19", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm and MiniMax Sparse Attention", "layerMix": "60 MiniMax Sparse Attention; 3 dense FFN + 57 MoE", "scale": "428B total, 23B active (5.4% active)", "totalParams": 428000000000, "activeParams": 23000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MiniMax Community License", "kvCacheText": "120 KiB", "kvCacheKiB": 120, "kvCacheLevel": "Moderate", "summary": "MiniMax M3 successor with sparse attention in all 60 decoder layers, quantization-aware training, and a DSpark draft head for speculative decoding.", "keyDetail": "All 60 layers use sparse attention; the first 3 retain dense FFNs, while the remaining 57 route to 4 of 128 experts plus 1 shared expert per token.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/minimax-m3.1-flash.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/minimax-m3.1-flash.webp", "alt": "Figure 1: MiniMax M3.1-Flash 428B", "width": 936, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash/blob/main/config.json", "title": "MiniMaxAI/MiniMax-M3.1-Flash" }, { "label": "License", "href": "https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2606.13392", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 28.9, "profile": { "Knowledge Work": 47.9, "Coding": 22.8, "Science": 14.4, "General": 30.5 }, "url": "https://artificialanalysis.ai/models/minimax-m3-1-flash" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 42.7, "profile": { "General": 40.1, "Scientific": 35.1, "Coding": 39.6, "Agents": 52.4 }, "url": "https://artificialanalysis.ai/models/minimax-m3-1-flash" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-minimax-m3-1-flash-428b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/minimax-m3-1-flash-428b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/minimax-m3-1-flash-428b.md" }, { "slug": "recursive-flint-40b-a2-5b", "order": 114, "name": "Recursive Flint (40B-A2.5B)", "baseTitle": "Recursive Flint 40B-A2.5B", "company": "Recursive", "country": "United States", "flag": "🇺🇸", "date": "2026-10-18", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm, learned attention sinks, and sliding-window/global attention", "layerMix": "35 sliding-window + 14 global GQA", "scale": "39.8B total, 2.5B active (6.3% active)", "totalParams": 39800000000, "activeParams": 2500000000, "context": "262,144 configured; 512k supported", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "98 KiB", "kvCacheKiB": 98, "kvCacheLevel": "Moderate", "summary": "Recursive's coding model uses two dense FFN layers followed by 47 MoE layers. The active-parameter count includes the output head.", "keyDetail": "Uses top-6 routing over 256 experts plus one shared expert, with RoPE in local layers, NoPE in global layers, and a local window covering the current token and 512 preceding tokens.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/recursive-flint.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/recursive-flint.webp", "alt": "Figure 1: Recursive Flint 40B-A2.5B", "width": 960, "height": 959 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/RecursiveSI/flint/blob/main/config.json", "title": "RecursiveSI/flint" }, { "label": "Tech report", "href": "https://recursive.com/updates/introducing-recursive-flint", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-recursive-flint-40b-a2-5b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/recursive-flint-40b-a2-5b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/recursive-flint-40b-a2-5b.md" }, { "slug": "reflection-beam-501b", "order": 112, "name": "Reflection Beam (501B)", "baseTitle": "Reflection Beam 501B", "company": "Reflection", "country": "United States", "flag": "🇺🇸", "date": "2026-10-16", "decoderType": "Sparse MoE", "attention": "Gated GQA with QK-Norm and sliding-window/global attention", "layerMix": "43 sliding-window + 9 global GQA", "scale": "501.3B total, 23.4B active (4.7% active)", "totalParams": 501300000000, "activeParams": 23400000000, "context": "1,048,576", "contextTokens": 1048576, "license": "Apache License 2.0", "kvCacheText": "208 KiB", "kvCacheKiB": 208, "kvCacheLevel": "High", "summary": "Text-only MoE decoder with three dense FFN layers followed by 49 expert layers. Each MoE layer activates 8 of 256 experts without a shared expert.", "keyDetail": "Combines 512-token local RoPE attention with global NoPE and query scaling. Uses sandwich RMSNorm, depth-scaled residuals, and a final logit softcap of 30.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/reflection-beam.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/reflection-beam.webp", "alt": "Figure 1: Reflection Beam 501B", "width": 960, "height": 944 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/config.json", "title": "reflection/Beam-1.0-501B-A23B" }, { "label": "License", "href": "https://huggingface.co/reflection/Beam-1.0-501B-A23B/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 17.7, "profile": { "Knowledge Work": 26.3, "Coding": 4.5, "Science": 10.9, "General": 29.2 }, "url": "https://artificialanalysis.ai/models/reflection-beam" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-reflection-beam-501b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/reflection-beam-501b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/reflection-beam-501b.md" }, { "slug": "dory1-0-4b", "order": 113, "name": "Dory1.0 (4B)", "baseTitle": "Dory1.0 4B", "company": "NVIDIA", "country": "United States", "flag": "🇺🇸", "date": "2026-10-16", "decoderType": "Dense", "attention": "Gated GQA; 3:1 sliding-window/full, RoPE", "layerMix": "8 input + 20 shared × 2 passes + 8 output (56 applications)", "scale": "4B parameters", "totalParams": 4000000000, "activeParams": 4000000000, "context": "262,144", "contextTokens": 262144, "license": "OpenMDW 1.1", "kvCacheText": "224 KiB", "kvCacheKiB": 224, "kvCacheLevel": "High", "summary": "NVIDIA's dense looped transformer applies a shared 20-block core twice between 8-block input and output stacks, using normalized GPT (nGPT) residual updates.", "keyDetail": "Reinjects the input-stack output before each recurrent pass. The 36 distinct blocks give 56 block applications by default, with L2-normalized residual updates and a 1,024-token attention window.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/dory-1-0-4b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/dory-1-0-4b.webp", "alt": "Figure 1: Dory1.0 4B", "width": 960, "height": 918 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16/blob/main/config.json", "title": "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16" }, { "label": "License", "href": "https://openmdw.ai/license/1-1/", "title": null } ], "concepts": [ { "slug": "looped-depth-sharing", "label": "Looped Transformer", "name": "Looped Transformer", "url": "https://llm-atlas-7or.pages.dev/concepts/looped-depth-sharing/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-dory1-0-4b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/dory1-0-4b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/dory1-0-4b.md" }, { "slug": "muse-spark-open-816b-a42b", "order": 110, "name": "Muse Spark Open (816B-A42B)", "baseTitle": "Muse Spark Open 816B-A42B", "company": "Meta", "country": "United States", "flag": "🇺🇸", "date": "2026-10-15", "decoderType": "Sparse MoE", "attention": "Gated GQA with scaleless QK-Norm, local-only RoPE, global NoPE, and 3:1 sliding-window/global attention", "layerMix": "46 sliding-window + 16 global gated GQA", "scale": "816B total, 42B active (5.1% active)", "totalParams": 816000000000, "activeParams": 42000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "Muse Model License", "kvCacheText": "248 KiB", "kvCacheKiB": 248, "kvCacheLevel": "High", "summary": "Meta's multimodal MoE model combines 62 latent-expert blocks with gated GQA, local-only RoPE, global NoPE, and learned residual gates; the figure focuses on the text decoder.", "keyDetail": "Selects 8 of 256 experts, compresses the expert path from 8,192 to 4,096 dimensions, normalizes each expert output before combining, and uses no shared expert.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/muse-spark-open.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/muse-spark-open.webp", "alt": "Figure 1: Muse Spark Open 816B-A42B", "width": 960, "height": 657 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B/blob/main/config.json", "title": "meta-models/Muse-Spark-Open-816B-A42B" }, { "label": "License", "href": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B/blob/main/LICENSE.md", "title": null } ], "concepts": [ { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "latent-moe", "label": "LatentMoE", "name": "Latent MoE", "url": "https://llm-atlas-7or.pages.dev/concepts/latent-moe/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 24.7, "profile": { "Knowledge Work": 37, "Coding": 7.8, "Science": 12.1, "General": 41.7 }, "url": "https://artificialanalysis.ai/models/muse-spark-open" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 38.9, "profile": { "General": 45, "Scientific": 29.4, "Coding": 33.3, "Agents": 42.8 }, "url": "https://artificialanalysis.ai/models/muse-spark-open" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-muse-spark-open-816b-a42b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/muse-spark-open-816b-a42b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/muse-spark-open-816b-a42b.md" }, { "slug": "nemotron-3-5-super-vl-120b-a12b", "order": 111, "name": "Nemotron 3.5 Super VL (120B-A12B)", "baseTitle": "Nemotron 3.5 Super VL 120B-A12B", "company": "NVIDIA", "country": "United States", "flag": "🇺🇸", "date": "2026-10-15", "decoderType": "Hybrid MoE", "attention": "Mostly Mamba-2 with eight GQA layers", "layerMix": "8 GQA + 40 Mamba-2 + 40 LatentMoE; GQA + MoE MTP predictor", "scale": "120B total, 12B active (10% active)", "totalParams": 120000000000, "activeParams": 12000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "OpenMDW 1.1", "kvCacheText": "8 KiB", "kvCacheKiB": 8, "kvCacheLevel": "Very low", "summary": "NVIDIA's vision-language model extends the Nemotron 3 Super backbone with C-RADIO vision encoding and shared-weight multi-token prediction. The figure shows the language backbone, with the vision encoder and MTP omitted.", "keyDetail": "Routes each token to 22 of 512 experts in a 1,024-dimensional latent space, alongside one shared expert at the full 4,096-dimensional width.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/nemotron-3-5-super-vl-120b-a12b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/nemotron-3-5-super-vl-120b-a12b.webp", "alt": "Figure 1: Nemotron 3.5 Super VL 120B-A12B", "width": 954, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/main/config.json", "title": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4" }, { "label": "License", "href": "https://openmdw.ai/license/1-1/", "title": null }, { "label": "Tech report", "href": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Super-and-Lightning-Technical-Report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "latent-moe", "label": "LatentMoE", "name": "Latent MoE", "url": "https://llm-atlas-7or.pages.dev/concepts/latent-moe/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 13.6, "profile": { "Knowledge Work": 19.5, "Coding": 3.7, "Science": 5.4, "General": 25.9 }, "url": "https://artificialanalysis.ai/models/nemotron-3-5-super-vl" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 26.5, "profile": { "General": 32.6, "Scientific": 20.6, "Coding": 23.9, "Agents": 26 }, "url": "https://artificialanalysis.ai/models/nemotron-3-5-super-vl" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-nemotron-3-5-super-vl-120b-a12b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/nemotron-3-5-super-vl-120b-a12b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/nemotron-3-5-super-vl-120b-a12b.md" }, { "slug": "ling-3-1-flash", "order": 109, "name": "Ling 3.1 Flash (560B)", "baseTitle": "Ling 3.1 Flash", "company": "InclusionAI", "country": "China", "flag": "🇨🇳", "date": "2026-10-13", "decoderType": "Sparse hybrid", "attention": "7:1 Kimi Delta Attention and gated MLA", "layerMix": "63 Kimi Delta Attention + 9 gated MLA + 1 MTP layer", "scale": "560B total, 25B active (4.5% active)", "totalParams": 560000000000, "activeParams": 25000000000, "context": "1M reported; 262,144 in config", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "10.1 KiB", "kvCacheKiB": 10.1, "kvCacheLevel": "Very low", "summary": "Hybrid MoE that extends Ling 3.0 Flash to 72 layers, with seven Kimi Delta Attention layers for every gated MLA layer.", "keyDetail": "Uses four initial dense FFN layers, top-8 routing over 512 experts plus one shared expert, and MLA with a 1,536-dimensional query bottleneck and 512-dimensional KV latent.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/ling-3-1-flash.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/ling-3-1-flash.webp", "alt": "Figure 1: Ling 3.1 Flash", "width": 960, "height": 946 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/inclusionAI/Ling-3.1-flash/blob/main/config.json", "title": "inclusionAI/Ling-3.1-flash" }, { "label": "Tech report", "href": "https://huggingface.co/inclusionAI/Ling-3.1-flash", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-ling-3-1-flash", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/ling-3-1-flash/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/ling-3-1-flash.md" }, { "slug": "kolibri-1-78b", "order": 108, "name": "Kolibri-1 (78B)", "baseTitle": "Kolibri-1 78B", "company": "Aleph Alpha", "country": "Germany", "flag": "🇩🇪", "date": "2026-10-03", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm and 4:1 sliding-window/global attention", "layerMix": "40 sliding-window + 10 global GQA", "scale": "78B total, 3.46B active (4.4% active)", "totalParams": 78000000000, "activeParams": 3460000000, "context": "262,144 native; 1,048,576 extrapolated", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "100 KiB", "kvCacheKiB": 100, "kvCacheLevel": "Moderate", "summary": "Aleph Alpha's German-English reasoning model uses 50 MoE blocks with a 4:1 mix of sliding-window and global GQA.", "keyDetail": "Uses top-6 routing over 384 experts plus one shared expert, with sandwich RMSNorm, RoPE in local layers, and NoPE in global layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/kolibri-1.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/kolibri-1.webp", "alt": "Figure 1: Kolibri-1 78B", "width": 960, "height": 951 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/Aleph-Alpha/Kolibri-1/blob/main/config.json", "title": "Aleph-Alpha/Kolibri-1" }, { "label": "Tech report", "href": "https://aleph-alpha.com/downloads/tech-report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-kolibri-1-78b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/kolibri-1-78b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/kolibri-1-78b.md" }, { "slug": "naive-n0-5-flash-309b", "order": 107, "name": "Naive-N0.5-Flash (309B)", "baseTitle": "Naive-N0.5-Flash 309B", "company": "NaiveAI", "country": "China", "flag": "🇨🇳", "date": "2026-09-27", "decoderType": "Sparse MoE", "attention": "GQA with sliding-window attention and DeepSeek Sparse Attention", "layerMix": "39 sliding-window + 9 GQA/DSA", "scale": "309B total, 15.5B active (5.0% active)", "totalParams": 309000000000, "activeParams": 15500000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "219.75 KiB", "kvCacheKiB": 219.75, "kvCacheLevel": "High", "summary": "MiMo-V2.5-derived sparse MoE that replaces its global attention layers with GQA-based DeepSeek Sparse Attention for a native 1M-token context.", "keyDetail": "Pairs a 128-token local window with top-2,048 DSA selection, four DSA KV heads, and a 16-head indexer. Uses 256 routed experts with top-8 routing and no full-attention layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/naive-n0-5-flash-309b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/naive-n0-5-flash-309b.webp", "alt": "Figure 1: Naive-N0.5-Flash 309B", "width": 960, "height": 935 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/NaiveAI/Naive-N0.5-Flash/blob/main/config.json", "title": "NaiveAI/Naive-N0.5-Flash" }, { "label": "License", "href": "https://huggingface.co/NaiveAI/Naive-N0.5-Flash/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://naive.ai/en/research/", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-naive-n0-5-flash-309b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/naive-n0-5-flash-309b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/naive-n0-5-flash-309b.md" }, { "slug": "ember-1", "order": 106, "name": "Ember-1 (2.8T)", "baseTitle": "Ember-1", "company": "Fireworks AI", "country": "United States", "flag": "🇺🇸", "date": "2026-09-23", "decoderType": "Sparse hybrid", "attention": "3:1 Kimi Delta Attention and Gated MLA with NoPE", "layerMix": "69 KDA + 24 Gated MLA", "scale": "2.8T total, 104B active (3.7% active)", "totalParams": 2800000000000, "activeParams": 104000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "N/A", "kvCacheText": "27 KiB", "kvCacheKiB": 27, "kvCacheLevel": "Low", "summary": "Fireworks' Kimi K3 derivative, post-trained for shorter reasoning traces. Architecture details follow the published Kimi K3 base model; the diagram focuses on its text backbone.", "keyDetail": "Inherits Kimi K3's eight Block Attention Residual groups and Stable LatentMoE, with one dense layer followed by 92 MoE layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/ember-1.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/ember-1.webp", "alt": "Figure 1: Ember-1", "width": 960, "height": 884 }, "articleUrl": null, "links": [ { "label": "Kimi K3 base config", "href": "https://huggingface.co/moonshotai/Kimi-K3/blob/main/config.json", "title": "moonshotai/Kimi-K3" }, { "label": "Tech report", "href": "https://fireworks.ai/blog/ember-1", "title": null } ], "concepts": [ { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "attention-residuals", "label": "Attention Residuals", "name": "Attention Residuals", "url": "https://llm-atlas-7or.pages.dev/concepts/attention-residuals/" }, { "slug": "latent-moe", "label": "LatentMoE", "name": "Latent MoE", "url": "https://llm-atlas-7or.pages.dev/concepts/latent-moe/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-ember-1", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/ember-1/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/ember-1.md" }, { "slug": "xiaomi-mimo-v2-6-flash-rl-309b", "order": 104, "name": "Xiaomi MiMo-V2.6-Flash-RL (309B)", "baseTitle": "Xiaomi MiMo-V2.6-Flash-RL 309B", "company": "Xiaomi", "country": "China", "flag": "🇨🇳", "date": "2026-09-21", "decoderType": "Sparse omnimodal MoE", "attention": "GQA with sliding-window/global attention", "layerMix": "39 sliding-window + 9 global", "scale": "309B total, 15B active (4.9% active)", "totalParams": 309000000000, "activeParams": 15000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "217.5 KiB", "kvCacheKiB": 217.5, "kvCacheLevel": "High", "summary": "Omnimodal sparse MoE model with the MiMo-V2.5 text backbone, mixed-domain reinforcement learning, and a five-layer DFlash speculative decoder.", "keyDetail": "Uses 256 routed experts with top-8 routing, one dense prefix layer, no shared experts, and a 128-token sliding window.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/mimo-2.6-flash.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/mimo-2.6-flash.webp", "alt": "Figure 1: Xiaomi MiMo-V2.6-Flash-RL 309B", "width": 960, "height": 949 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/blob/main/config.json", "title": "XiaomiMiMo/MiMo-V2.6-Flash-RL" }, { "label": "Tech report", "href": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL/resolve/main/MiMo_V2_6_technical_report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 26.7, "profile": { "Knowledge Work": 44.2, "Coding": 22.1, "Science": 14.4, "General": 25.9 }, "url": "https://artificialanalysis.ai/models/mimo-v2-6-flash" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 37.9, "profile": { "General": 32, "Scientific": 23.5, "Coding": 37, "Agents": 53.9 }, "url": "https://artificialanalysis.ai/models/mimo-v2-6-flash" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-xiaomi-mimo-v2-6-flash-rl-309b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-6-flash-rl-309b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-6-flash-rl-309b.md" }, { "slug": "xiaomi-mimo-v2-6-pro-rl-1-02t", "order": 105, "name": "Xiaomi MiMo-V2.6-Pro-RL (1.02T)", "baseTitle": "Xiaomi MiMo-V2.6-Pro-RL 1.02T", "company": "Xiaomi", "country": "China", "flag": "🇨🇳", "date": "2026-09-21", "decoderType": "Sparse omnimodal MoE", "attention": "GQA with 6:1 sliding-window/global attention", "layerMix": "60 sliding-window + 10 global", "scale": "1.02T total, 42B active (4.1% active)", "totalParams": 1020000000000, "activeParams": 42000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "350 KiB", "kvCacheKiB": 350, "kvCacheLevel": "Very high", "summary": "Omnimodal sparse MoE model with the MiMo-V2.5-Pro text backbone, mixed-domain reinforcement learning, and a five-layer DFlash speculative decoder.", "keyDetail": "Uses 384 routed experts with top-8 routing, one dense prefix layer, no shared experts, and a 128-token sliding window.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/mimo-2.6-pro.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/mimo-2.6-pro.webp", "alt": "Figure 1: Xiaomi MiMo-V2.6-Pro-RL 1.02T", "width": 960, "height": 949 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL/blob/main/config.json", "title": "XiaomiMiMo/MiMo-V2.6-Pro-RL" }, { "label": "Tech report", "href": "https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL/resolve/main/MiMo_V2_6_technical_report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 29.9, "profile": { "Knowledge Work": 45.9, "Coding": 28.3, "Science": 17, "General": 28 }, "url": "https://artificialanalysis.ai/models/mimo-v2-6-pro" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 46.3, "profile": { "General": 42.3, "Scientific": 38, "Coding": 47.9, "Agents": 54.9 }, "url": "https://artificialanalysis.ai/models/mimo-v2-6-pro" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-xiaomi-mimo-v2-6-pro-rl-1-02t", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-6-pro-rl-1-02t/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-6-pro-rl-1-02t.md" }, { "slug": "step-5-preview-600b", "order": 103, "name": "Step-5 Preview (600B)", "baseTitle": "Step-5 Preview 600B", "company": "StepFun", "country": "China", "flag": "🇨🇳", "date": "2026-09-20", "decoderType": "Sparse MoE", "attention": "Gated GQA with QK-Norm and 3:1 sliding-window/sparse attention", "layerMix": "69 sliding-window + 23 sparse GQA + 3 MTP layers", "scale": "600B total, 27B active (4.5% active)", "totalParams": 600000000000, "activeParams": 27000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "N/A", "kvCacheText": null, "kvCacheKiB": null, "kvCacheLevel": null, "summary": "StepFun's 92-layer MoE pairs 512-token sliding-window attention with sparse GQA for a 1M-token context window; the figure shows the text decoder.", "keyDetail": "Uses top-512 sparse indexing with block size 8, 352 routed experts with 8 active plus one shared expert, and MTP-3. The first three and final backbone layers use dense FFNs with intermediate size 13,824.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/step-5.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/step-5.webp", "alt": "Figure 1: Step-5 Preview 600B", "width": 960, "height": 940 }, "articleUrl": null, "links": [ { "label": "config.json (mirror)", "href": "https://huggingface.co/SHSLab/Step-5-Preview-BF16/blob/main/config.json", "title": "SHSLab/Step-5-Preview-BF16" }, { "label": "Tech report", "href": "https://www.stepfun.com/step-5-preview", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 28.5, "profile": { "Knowledge Work": 40.9, "Coding": 20.5, "Science": 14.4, "General": 38 }, "url": "https://artificialanalysis.ai/models/step-5" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 43.7, "profile": { "General": 43, "Scientific": 33.7, "Coding": 46.1, "Agents": 49.7 }, "url": "https://artificialanalysis.ai/models/step-5" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-step-5-preview-600b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/step-5-preview-600b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/step-5-preview-600b.md" }, { "slug": "deepseek-v4-1-flash", "order": 102, "name": "DeepSeek V4.1-Flash (552B)", "baseTitle": "DeepSeek V4.1-Flash", "company": "DeepSeek", "country": "China", "flag": "🇨🇳", "date": "2026-09-10", "decoderType": "Sparse MoE (causal encoder-decoder)", "attention": "MLA-style CSA2 + SWA with cross-layer KV and sparse-index reuse", "layerMix": "20 encoder + 20 decoder layers; 2 SWA-only + 38 CSA2/SWA", "scale": "552B backbone + 196B Engram; 8B active prefill, 16B active decode (2.9% of backbone)", "totalParams": 552000000000, "activeParams": 8000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "3.125 KiB", "kvCacheKiB": 3.125, "kvCacheLevel": "Very low", "summary": "Causal encoder-decoder MoE with CSA2 cache sharing, Engram memory, and a million-token context. The figure shows the text backbone.", "keyDetail": "Decoder global KV comes from final encoder states. Single-Pass mHC mixes four residual streams around each attention and MoE sublayer.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/deepseek-v4-1-flash.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/deepseek-v4-1-flash.webp", "alt": "Figure 1: DeepSeek V4.1-Flash", "width": 960, "height": 683 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/main/config.json", "title": "deepseek-ai/DeepSeek-V4.1-Flash" }, { "label": "License", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/main/DeepSeek_V41_Tech_Report.pdf", "title": null } ], "concepts": [ { "slug": "mhc", "label": "mHC", "name": "Manifold-constrained Hyper-Connections", "url": "https://llm-atlas-7or.pages.dev/concepts/mhc/" }, { "slug": "kv-sharing", "label": "KV Sharing", "name": "Cross-Layer KV Sharing", "url": "https://llm-atlas-7or.pages.dev/concepts/kv-sharing/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 39.5, "profile": { "General": 34.3, "Scientific": 26.8, "Coding": 39.3, "Agents": 52.8 }, "url": "https://artificialanalysis.ai/models/deepseek-v4-1-flash" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-deepseek-v4-1-flash", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/deepseek-v4-1-flash/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/deepseek-v4-1-flash.md" }, { "slug": "tencent-hy4-preview-770b-a49b", "order": 101, "name": "Tencent Hy4-preview (770B-A49B)", "baseTitle": "Tencent Hy4-preview 770B-A49B", "company": "Tencent", "country": "China", "flag": "🇨🇳", "date": "2026-08-28", "decoderType": "Sparse MoE", "attention": "Gated MLA with DeepSeek Sparse Attention and IndexCache", "layerMix": "78 gated MLA/DSA + 1 MTP layer", "scale": "770B backbone, 49B active (6.4% active); plus 10B MTP", "totalParams": 770000000000, "activeParams": 49000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "Apache License 2.0", "kvCacheText": "87.8 KiB", "kvCacheKiB": 87.8, "kvCacheLevel": "Moderate", "summary": "Tencent's 770B sparse MoE combines gated DeepSeek Sparse Attention, cross-layer index reuse, and four-stream identity Hyper-Connections.", "keyDetail": "Uses 256 routed experts with 8 routed plus 1 shared expert active per token, a top-2,048 DSA indexer, cross-layer index reuse, and four iHC residual streams.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/tencent-hy4.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/tencent-hy4.webp", "alt": "Figure 1: Tencent Hy4-preview 770B-A49B", "width": 960, "height": 739 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/tencent/Hy4-preview/blob/main/config.json", "title": "tencent/Hy4-preview" }, { "label": "License", "href": "https://huggingface.co/tencent/Hy4-preview/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "mhc", "label": "mHC", "name": "Manifold-constrained Hyper-Connections", "url": "https://llm-atlas-7or.pages.dev/concepts/mhc/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "indexshare", "label": "IndexShare", "name": "IndexShare", "url": "https://llm-atlas-7or.pages.dev/concepts/indexshare/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-tencent-hy4-preview-770b-a49b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/tencent-hy4-preview-770b-a49b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/tencent-hy4-preview-770b-a49b.md" }, { "slug": "glm-5-3-flash", "order": 99, "name": "GLM-5.3-Flash (320B)", "baseTitle": "GLM-5.3-Flash", "company": "Z.ai", "country": "China", "flag": "🇨🇳", "date": "2026-08-26", "decoderType": "Sparse hybrid", "attention": "3:1 Kimi Delta Attention and MLA with DeepSeek Sparse Attention", "layerMix": "34 KDA + 11 MLA/DSA + 1 MTP layer", "scale": "320B total, 18B active (5.6% active)", "totalParams": 320000000000, "activeParams": 18000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "11 KiB", "kvCacheKiB": 11, "kvCacheLevel": "Very low", "summary": "Natively multimodal GLM refresh, previously previewed as Ox Alpha, that replaces most MLA/DSA layers with Kimi Delta Attention and adds four-stream mHC.", "keyDetail": "Mixes 34 Kimi Delta Attention layers with 11 MLA/DSA layers and routes residuals through four mHC streams.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/glm-5.3-flash.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/glm-5.3-flash.webp", "alt": "Figure 1: GLM-5.3-Flash", "width": 960, "height": 804 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/main/config.json", "title": "zai-org/GLM-5.3-Flash" }, { "label": "License", "href": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2602.15763", "title": null } ], "concepts": [ { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "mhc", "label": "mHC", "name": "Manifold-constrained Hyper-Connections", "url": "https://llm-atlas-7or.pages.dev/concepts/mhc/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 27.8, "profile": { "Knowledge Work": 43.7, "Coding": 23.7, "Science": 12.5, "General": 30.9 }, "url": "https://artificialanalysis.ai/models/glm-5-3-flash" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 41.8, "profile": { "General": 39.7, "Scientific": 27.6, "Coding": 42.2, "Agents": 53 }, "url": "https://artificialanalysis.ai/models/glm-5-3-flash" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-glm-5-3-flash", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/glm-5-3-flash/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/glm-5-3-flash.md" }, { "slug": "qwen3-8-flash-next-125b-a6b", "order": 100, "name": "Qwen3.8-Flash-Next (125B-A6B)", "baseTitle": "Qwen3.8-Flash-Next 125B-A6B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2026-08-26", "decoderType": "Sparse hybrid", "attention": "3:1 Gated DeltaNet and Qwen Sparse Attention (QSA)", "layerMix": "12 QSA + 36 DeltaNet", "scale": "125B core, 6B active (4.8% active); plus 51B n-gram embeddings and 4B MTP", "totalParams": 125000000000, "activeParams": 6000000000, "context": "262,144", "contextTokens": 262144, "license": "Qwen Community License 1.0", "kvCacheText": "24 KiB", "kvCacheKiB": 24, "kvCacheLevel": "Very low", "summary": "Experimental vision-language MoE whose text backbone combines Gated DeltaNet, micro-block QSA, four-branch gated residuals, and n-gram embeddings; the figure omits the vision encoder and MTP head.", "keyDetail": "Replaces full attention with micro-block QSA while using four gated residual branches and a 20M-entry n-gram embedding at layer 2.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-8-flash-next.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-8-flash-next.webp", "alt": "Figure 1: Qwen3.8-Flash-Next 125B-A6B", "width": 960, "height": 798 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/blob/main/config.json", "title": "Qwen/Qwen3.8-Flash-Next" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://github.com/QwenLM/Qwen3.8-Flash-Next/blob/main/tech_report.pdf", "title": null } ], "concepts": [ { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "qwen-sparse-attention", "label": "Qwen Sparse Attention", "name": "Qwen Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/qwen-sparse-attention/" }, { "slug": "gated-residuals", "label": "Gated Residuals", "name": "Gated Residuals", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-residuals/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "per-layer-embeddings", "label": "PLE", "name": "Per-Layer Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/per-layer-embeddings/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": null, "profile": { "General": 26.6 }, "url": null }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 39.8, "profile": { "General": 35.8, "Scientific": 24.6, "Coding": 37.9, "Agents": 55.1 }, "url": "https://artificialanalysis.ai/models/qwen3-8-flash-next" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-8-flash-next-125b-a6b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-8-flash-next-125b-a6b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-8-flash-next-125b-a6b.md" }, { "slug": "ornith-1-5-35b-a3b", "order": 98, "name": "Ornith 1.5 (35B-A3B)", "baseTitle": "Ornith 1.5 35B-A3B", "company": "Ornith", "country": "United States", "flag": "🇺🇸", "date": "2026-08-23", "decoderType": "Sparse hybrid", "attention": "3:1 Gated DeltaNet and Gated Attention", "layerMix": "10 gated attention + 30 DeltaNet", "scale": "35B total, 3B active (8.6% active)", "totalParams": 35000000000, "activeParams": 3000000000, "context": "262,144", "contextTokens": 262144, "license": "MIT License", "kvCacheText": "20 KiB", "kvCacheKiB": 20, "kvCacheLevel": "Very low", "summary": "Uses the same sparse-hybrid decoder geometry as Qwen3.6 35B-A3B, paired with Ornith's self-improvement training recipe.", "keyDetail": "Matches Qwen3.6 35B-A3B's 40-layer decoder geometry, with 256 routed experts, top-8 routing, and one shared expert.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/ornith-1-5-35b-a3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/ornith-1-5-35b-a3b.webp", "alt": "Figure 1: Ornith 1.5 35B-A3B", "width": 960, "height": 904 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B/blob/main/config.json", "title": "ornith-ai/Ornith-1.5-35B-A3B" }, { "label": "Tech report", "href": "https://ornith.ai/ornith_1_5.html", "title": null } ], "concepts": [ { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-ornith-1-5-35b-a3b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/ornith-1-5-35b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/ornith-1-5-35b-a3b.md" }, { "slug": "qwen3-8-27b", "order": 97, "name": "Qwen3.8 (27B)", "baseTitle": "Qwen3.8 27B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2026-08-14", "decoderType": "Dense hybrid", "attention": "3:1 Gated DeltaNet and Gated Attention", "layerMix": "16 gated attention + 48 DeltaNet", "scale": "27B parameters", "totalParams": 27000000000, "activeParams": 27000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "64 KiB", "kvCacheKiB": 64, "kvCacheLevel": "Low", "summary": "Dense 27B native vision-language model with a 64-layer Gated DeltaNet and Gated Attention language backbone; the figure omits the vision encoder and MTP head.", "keyDetail": "Uses a 64-layer 3:1 Gated DeltaNet/Gated Attention decoder with partial RoPE and one MTP layer.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-8-27b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-8-27b.webp", "alt": "Figure 1: Qwen3.8 27B", "width": 960, "height": 915 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/main/config.json", "title": "Qwen/Qwen3.8-27B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 20.1, "profile": { "Knowledge Work": 39.7, "Coding": 11.2, "Science": 1.9, "General": 27.3 }, "url": "https://artificialanalysis.ai/models/qwen3-8-27b" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 33.7, "profile": { "General": 35.9, "Scientific": 19.7, "Coding": 26.1, "Agents": 45.8 }, "url": "https://artificialanalysis.ai/models/qwen3-8-27b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-8-27b", "family": "hybrid", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-8-27b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-8-27b.md" }, { "slug": "nemotron-3-5-lightning-30b-a3b", "order": 96, "name": "Nemotron 3.5 Lightning (30B-A3B)", "baseTitle": "Nemotron 3.5 Lightning 30B-A3B", "company": "NVIDIA", "country": "United States", "flag": "🇺🇸", "date": "2026-08-11", "decoderType": "Hybrid MoE", "attention": "Mostly Mamba-2 with six GQA layers", "layerMix": "6 GQA + 23 Mamba-2 + 23 MoE; GQA + MoE MTP predictor", "scale": "30B total, 3B active (10% active)", "totalParams": 30000000000, "activeParams": 3000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "OpenMDW 1.1", "kvCacheText": "6 KiB", "kvCacheKiB": 6, "kvCacheLevel": "Very low", "summary": "NVIDIA's 30B hybrid model keeps 3B parameters active and extends the Nemotron 3 Nano backbone with a 1M-token context window and multi-token prediction.", "keyDetail": "Adds a shared GQA-plus-MoE MTP predictor to the 52-layer Nemotron 3 Nano backbone.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/nemotron-3-5-lightning-30b-a3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/nemotron-3-5-lightning-30b-a3b.webp", "alt": "Figure 1: Nemotron 3.5 Lightning 30B-A3B", "width": 937, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4/blob/main/config.json", "title": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" }, { "label": "License", "href": "https://openmdw.ai/license/1-1/", "title": null }, { "label": "Tech report", "href": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Super-and-Lightning-Technical-Report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 5.5, "profile": { "Knowledge Work": 1, "Coding": 0.2, "Science": 0.6, "General": 20.6 }, "url": "https://artificialanalysis.ai/models/nemotron-3-5-lightning" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 12.9, "profile": { "General": 26.3, "Scientific": 5.3, "Coding": 16.3, "Agents": 2.6 }, "url": "https://artificialanalysis.ai/models/nemotron-3-5-lightning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-nemotron-3-5-lightning-30b-a3b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/nemotron-3-5-lightning-30b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/nemotron-3-5-lightning-30b-a3b.md" }, { "slug": "muse-glimmer-30b", "order": 95, "name": "Muse Glimmer (30B)", "baseTitle": "Muse Glimmer 30B", "company": "Meta", "country": "United States", "flag": "🇺🇸", "date": "2026-08-10", "decoderType": "Dense", "attention": "Gated GQA with scaleless QK-Norm, local-only RoPE, global NoPE, and 3:1 sliding-window/global attention", "layerMix": "39 sliding-window + 13 global gated GQA", "scale": "29.6B parameters (including perception encoder)", "totalParams": 29600000000, "activeParams": 29600000000, "context": "131,072+", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "52 KiB", "kvCacheKiB": 52, "kvCacheLevel": "Low", "summary": "Meta's dense 29.6B multimodal model combines a 52-layer sandwich-norm decoder with gated GQA and a separate perception encoder; the figure focuses on the text backbone.", "keyDetail": "Applies RoPE only in 2,048-token local layers, uses NoPE in global layers, and gates the attention output before projection.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/muse-glimmer-30b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/muse-glimmer-30b.webp", "alt": "Figure 1: Muse Glimmer 30B", "width": 889, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/main/config.json", "title": "meta-models/Muse-Glimmer-30B" }, { "label": "License", "href": "https://huggingface.co/meta-models/Muse-Glimmer-30B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://research.meta.ai/static/muse-glimmer-methodology", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 6.3, "profile": { "Knowledge Work": 3.4, "Coding": 1.1, "Science": 2.8, "General": 18.1 }, "url": "https://artificialanalysis.ai/models/muse-glimmer" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 17.5, "profile": { "General": 29.2, "Scientific": 12.3, "Coding": 22.7, "Agents": 5.9 }, "url": "https://artificialanalysis.ai/models/muse-glimmer" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-muse-glimmer-30b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/muse-glimmer-30b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/muse-glimmer-30b.md" }, { "slug": "ling-3-0-flash", "order": 94, "name": "Ling 3.0 Flash (124B)", "baseTitle": "Ling 3.0 Flash", "company": "InclusionAI", "country": "China", "flag": "🇨🇳", "date": "2026-08-04", "decoderType": "Sparse hybrid", "attention": "5:1 Kimi Delta Attention and gated MLA", "layerMix": "35 Kimi Delta Attention + 7 gated MLA + 1 MTP layer", "scale": "124B total, 5.1B active (4.1% active)", "totalParams": 124000000000, "activeParams": 5100000000, "context": "262,144", "contextTokens": 262144, "license": "MIT License", "kvCacheText": "7.9 KiB", "kvCacheKiB": 7.9, "kvCacheLevel": "Very low", "summary": "Hybrid MoE that alternates five Kimi Delta Attention layers with one gated MLA layer.", "keyDetail": "Uses fine-grained diagonal KDA gating, top-8 routing over 512 experts plus one shared expert, and one MTP layer.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/ling-3-0-flash.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/ling-3-0-flash.webp", "alt": "Figure 1: Ling 3.0 Flash", "width": 960, "height": 923 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/main/config.json", "title": "inclusionAI/Ling-3.0-flash" }, { "label": "Tech report", "href": "https://huggingface.co/inclusionAI/Ling-3.0-flash", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 20.1, "profile": { "General": 29.3, "Scientific": 12.7, "Coding": 21, "Agents": 15.4 }, "url": "https://artificialanalysis.ai/models/ling-3-0-flash" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-ling-3-0-flash", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/ling-3-0-flash/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/ling-3-0-flash.md" }, { "slug": "kimi-k3", "order": 93, "name": "Kimi K3 (2.8T)", "baseTitle": "Kimi K3", "company": "Moonshot AI", "country": "China", "flag": "🇨🇳", "date": "2026-07-27", "decoderType": "Sparse hybrid", "attention": "3:1 Kimi Delta Attention and Gated MLA with NoPE", "layerMix": "69 KDA + 24 Gated MLA", "scale": "2.8T total, 104B active (3.7% active)", "totalParams": 2800000000000, "activeParams": 104000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "Kimi K3 License", "kvCacheText": "27 KiB", "kvCacheKiB": 27, "kvCacheLevel": "Low", "summary": "2.8T sparse hybrid that combines a 3:1 KDA and Gated MLA stack with Block Attention Residuals and Stable LatentMoE. It is also native multimodal, but the figure focuses on the text portion.", "keyDetail": "Uses eight Block Attention Residual groups and Stable LatentMoE with 16 of 896 routed experts active per token.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/kimi-k3.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/kimi-k3.webp", "alt": "Figure 1: Kimi K3", "width": 960, "height": 884 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/moonshotai/Kimi-K3/blob/main/config.json", "title": "moonshotai/Kimi-K3" }, { "label": "License", "href": "https://huggingface.co/moonshotai/Kimi-K3/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://github.com/MoonshotAI/Kimi-K3/blob/main/k3_tech_report.pdf", "title": null } ], "concepts": [ { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "attention-residuals", "label": "Attention Residuals", "name": "Attention Residuals", "url": "https://llm-atlas-7or.pages.dev/concepts/attention-residuals/" }, { "slug": "latent-moe", "label": "LatentMoE", "name": "Latent MoE", "url": "https://llm-atlas-7or.pages.dev/concepts/latent-moe/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": null, "profile": { "Science": 20.7 }, "url": null }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 43.6, "profile": { "General": 45.8, "Scientific": 35.2, "Coding": 36.1, "Agents": 51.9 }, "url": "https://artificialanalysis.ai/models/kimi-k3/" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-kimi-k3", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/kimi-k3/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/kimi-k3.md" }, { "slug": "solar-open-2", "order": 92, "name": "Solar Open 2 (250B)", "baseTitle": "Solar Open 2", "company": "Upstage", "country": "South Korea", "flag": "🇰🇷", "date": "2026-07-22", "decoderType": "Sparse hybrid", "attention": "3:1 Kimi Delta Attention and gated GQA with NoPE", "layerMix": "12 gated GQA + 36 Kimi Delta Attention", "scale": "250B total, 15B active (6% active)", "totalParams": 250000000000, "activeParams": 15000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "Upstage Solar License", "kvCacheText": "48 KiB", "kvCacheKiB": 48, "kvCacheLevel": "Low", "summary": "Upstage's sparse hybrid alternates one gated GQA layer with three Kimi Delta Attention layers and reaches a 1M-token context without positional embeddings.", "keyDetail": "Uses a repeating gated-GQA/KDA hybrid stack, kernel-4 ShortConv in KDA, and top-8 routing over 320 experts plus one shared expert.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/solar-open-2.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/solar-open-2.webp", "alt": "Figure 1: Solar Open 2", "width": 960, "height": 714 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/upstage/Solar-Open2-250B/blob/main/config.json", "title": "upstage/Solar-Open2-250B" }, { "label": "License", "href": "https://huggingface.co/upstage/Solar-Open2-250B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://huggingface.co/upstage/Solar-Open2-250B/blob/main/Solar_Open_2_Tech_Report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 24.7, "profile": { "Scientific": 17.1 }, "url": "https://artificialanalysis.ai/models/solar-open2-250b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-solar-open-2", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/solar-open-2/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/solar-open-2.md" }, { "slug": "antares-1b", "order": 88, "name": "Antares (1B)", "baseTitle": "Antares 1B", "company": "Cisco", "country": "United States", "flag": "🇺🇸", "date": "2026-07-21", "decoderType": "Dense", "attention": "GQA with RoPE", "layerMix": "40 GQA", "scale": "1B parameters", "totalParams": 1000000000, "activeParams": 1000000000, "context": "131,072", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "80 KiB", "kvCacheKiB": 80, "kvCacheLevel": "Moderate", "summary": "Cisco Foundation AI's compact terminal agent uses a dense Granite 4.0 backbone and is post-trained with SFT and GRPO for vulnerability localization.", "keyDetail": "Retains Granite 4.0's MuP scaling multipliers and tied input/output embeddings, then adds SFT and GRPO for terminal-based vulnerability localization.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/antares-1b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/antares-1b.webp", "alt": "Figure 1: Antares 1B", "width": 919, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/fdtn-ai/antares-1b/blob/main/config.json", "title": "fdtn-ai/antares-1b" }, { "label": "Tech report", "href": "https://cisco-foundation-ai.github.io/antares/technical-report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-antares-1b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/antares-1b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/antares-1b.md" }, { "slug": "laguna-s-2-1", "order": 89, "name": "Laguna S 2.1 (118B)", "baseTitle": "Laguna S 2.1", "company": "Poolside", "country": "United States", "flag": "🇺🇸", "date": "2026-07-21", "decoderType": "Sparse MoE", "attention": "Gated GQA with QK-Norm and 3:1 sliding-window/global attention", "layerMix": "36 sliding-window + 12 global GQA", "scale": "118B total, 8B active (6.8% active)", "totalParams": 118000000000, "activeParams": 8000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "OpenMDW License 1.1", "kvCacheText": "192 KiB", "kvCacheKiB": 192, "kvCacheLevel": "High", "summary": "Poolside's 118B sparse MoE scales the Laguna XS recipe to 48 layers and 1M context while activating about 8B parameters per token.", "keyDetail": "Uses 256 routed experts with top-10 routing plus one shared expert, a dense first layer, and per-layer query-head counts.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/laguna-s-2-1.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/laguna-s-2-1.webp", "alt": "Figure 1: Laguna S 2.1", "width": 960, "height": 913 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/poolside/Laguna-S-2.1/blob/main/config.json", "title": "poolside/Laguna-S-2.1" }, { "label": "License", "href": "https://huggingface.co/poolside/Laguna-S-2.1/blob/main/LICENSE.md", "title": null }, { "label": "Tech report", "href": "https://poolside.ai/blog/introducing-laguna-s-2-1", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "attention-budgeting", "label": "Attention Budgeting", "name": "Layer-wise attention budgeting", "url": "https://llm-atlas-7or.pages.dev/concepts/attention-budgeting/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-laguna-s-2-1", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/laguna-s-2-1/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/laguna-s-2-1.md" }, { "slug": "motif-3-beta", "order": 90, "name": "Motif 3 Beta (314B)", "baseTitle": "Motif 3 Beta", "company": "Motif Technologies", "country": "South Korea", "flag": "🇰🇷", "date": "2026-07-21", "decoderType": "Sparse MoE", "attention": "GDLA with MLA-style latent projections, RoPE, and 3:1 sliding-window/full attention", "layerMix": "39 sliding-window + 14 full GDLA", "scale": "314B total, 13B active (4.1% active)", "totalParams": 314000000000, "activeParams": 13000000000, "context": "262,144", "contextTokens": 262144, "license": "Motif Technologies Non-Commercial License", "kvCacheText": "530 KiB", "kvCacheKiB": 530, "kvCacheLevel": "Very high", "summary": "Large sparse MoE that combines Grouped Differential Latent Attention, per-expert PolyNorm, and modified mHC in a 53-layer decoder.", "keyDetail": "Routes each token through 8 of 384 experts plus one shared expert, with per-expert PolyNorm and a 3:1 sliding-window/full GDLA pattern.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/motif-3-beta.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/motif-3-beta.webp", "alt": "Figure 1: Motif 3 Beta", "width": 960, "height": 798 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/Motif-Technologies/Motif-3-Beta/blob/main/config.json", "title": "Motif-Technologies/Motif-3-Beta" }, { "label": "License", "href": "https://huggingface.co/Motif-Technologies/Motif-3-Beta/blob/main/README.md#license", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "mhc", "label": "mHC", "name": "Manifold-constrained Hyper-Connections", "url": "https://llm-atlas-7or.pages.dev/concepts/mhc/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "polynorm", "label": "PolyNorm", "name": "PolyNorm", "url": "https://llm-atlas-7or.pages.dev/concepts/polynorm/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 32.3, "profile": { "Scientific": 23.2 }, "url": "https://artificialanalysis.ai/models/motif-0714" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-motif-3-beta", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/motif-3-beta/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/motif-3-beta.md" }, { "slug": "nanbeige-4-2-3b", "order": 91, "name": "Nanbeige 4.2 (3B)", "baseTitle": "Nanbeige 4.2 3B", "company": "BOSS Zhipin", "country": "China", "flag": "🇨🇳", "date": "2026-07-21", "decoderType": "Dense", "attention": "GQA with RoPE", "layerMix": "22 shared GQA layers × 2 passes", "scale": "4B total, 3B non-embedding", "totalParams": 4000000000, "activeParams": 4000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "176 KiB", "kvCacheKiB": 176, "kvCacheLevel": "High", "summary": "Compact agentic model that runs the same 22-layer dense transformer stack twice, resulting in 44 transformer blocks (layers) without duplicating the layer weights.", "keyDetail": "Reuses the same 22-layer stack for a second pass, giving 44 block applications with 3B non-embedding parameters.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/nanbeige-4-2-3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/nanbeige-4-2-3b.webp", "alt": "Figure 1: Nanbeige 4.2 3B", "width": 960, "height": 953 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/Nanbeige/Nanbeige4.2-3B/blob/main/config.json", "title": "Nanbeige/Nanbeige4.2-3B" }, { "label": "Tech report", "href": "https://huggingface.co/Nanbeige/Nanbeige4.2-3B/blob/main/Nanbeige42_report.pdf", "title": null } ], "concepts": [ { "slug": "looped-depth-sharing", "label": "Looped Transformer", "name": "Looped Transformer", "url": "https://llm-atlas-7or.pages.dev/concepts/looped-depth-sharing/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-nanbeige-4-2-3b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/nanbeige-4-2-3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/nanbeige-4-2-3b.md" }, { "slug": "btl-3-27b", "order": 87, "name": "BTL-3 (27B)", "baseTitle": "BTL-3 27B", "company": "Bad Theory Labs", "country": "Nigeria", "flag": "🇳🇬", "date": "2026-07-20", "decoderType": "Dense hybrid", "attention": "3:1 Gated DeltaNet and Gated Attention", "layerMix": "16 gated attention + 48 DeltaNet", "scale": "27B base + 934 MB LoRA adapter", "totalParams": 27000000000, "activeParams": 27000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "64 KiB", "kvCacheKiB": 64, "kvCacheLevel": "Low", "summary": "Agent-oriented Qwen3.6-27B derivative that adds a rank-32 LoRA adapter for coding, structured tool use, and long multi-turn execution.", "keyDetail": "Retains Qwen3.6-27B's 64-layer hybrid stack while adapting its attention, DeltaNet, and FFN projections through rank-32 LoRA.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/btl-3-27b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/btl-3-27b.webp", "alt": "Figure 1: BTL-3 27B", "width": 960, "height": 904 }, "articleUrl": null, "links": [ { "label": "adapter_config.json", "href": "https://huggingface.co/badtheorylabs/BTL-3/blob/main/adapter_config.json", "title": "badtheorylabs/BTL-3" }, { "label": "License", "href": "https://huggingface.co/badtheorylabs/BTL-3/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-btl-3-27b", "family": "hybrid", "url": "https://llm-atlas-7or.pages.dev/models/btl-3-27b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/btl-3-27b.md" }, { "slug": "inkling", "order": 86, "name": "Inkling (975B)", "baseTitle": "Inkling", "company": "Thinking Machines Lab", "country": "United States", "flag": "🇺🇸", "date": "2026-07-15", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm and learned relative-position bias in a 5:1 sliding-window/global pattern", "layerMix": "55 sliding-window + 11 global GQA", "scale": "975B total, 41B active (4.2% active)", "totalParams": 975000000000, "activeParams": 41000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "Apache License 2.0", "kvCacheText": "484 KiB", "kvCacheKiB": 484, "kvCacheLevel": "Very high", "summary": "Thinking Machines Lab's native multimodal model combines a 66-layer sparse MoE decoder with a 5:1 mix of local and global GQA.", "keyDetail": "Uses learned relative-position bias instead of RoPE and adds four short convolutions to each decoder layer.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/inkling.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/inkling.webp", "alt": "Figure 1: Inkling", "width": 930, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/thinkingmachines/Inkling/blob/main/config.json", "title": "thinkingmachines/Inkling" }, { "label": "License", "href": "https://www.apache.org/licenses/LICENSE-2.0", "title": null }, { "label": "Tech report", "href": "https://thinkingmachines.ai/news/introducing-inkling/", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 12.5, "profile": { "Knowledge Work": 13.5, "Coding": 2.8, "Science": 7.2, "General": 26.5 }, "url": "https://artificialanalysis.ai/models/inkling" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 25, "profile": { "General": 36.4, "Scientific": 18.7, "Coding": 24, "Agents": 18.7 }, "url": "https://artificialanalysis.ai/models/inkling" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-inkling", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/inkling/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/inkling.md" }, { "slug": "soofi-s-30b-a3b", "order": 85, "name": "Soofi S (30B-A3B)", "baseTitle": "Soofi S 30B-A3B", "company": "SOOFI Consortium", "country": "Germany", "flag": "🇩🇪", "date": "2026-07-10", "decoderType": "Hybrid MoE", "attention": "Mostly Mamba-2 with 6 GQA layers and no positional embeddings (NoPE)", "layerMix": "6 GQA + 23 Mamba-2 + 23 MoE", "scale": "31.6B total, 3.2B active (10.1% active)", "totalParams": 31600000000, "activeParams": 3200000000, "context": "1,048,576", "contextTokens": 1048576, "license": "Custom license (preview)", "kvCacheText": "6 KiB", "kvCacheKiB": 6, "kvCacheLevel": "Very low", "summary": "German-English base model that retrains the Nemotron 3 Nano architecture on 26.68T tokens with a fully documented data mixture.", "keyDetail": "Reuses the Nemotron 3 Nano architecture without modification, isolating the effect of a German-English data and training recipe.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/soofi.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/soofi.webp", "alt": "Figure 1: Soofi S 30B-A3B", "width": 944, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/Soofi-Project/Soofi-S-Base/blob/main/config.json", "title": "Soofi-Project/Soofi-S-Base" }, { "label": "License", "href": "https://huggingface.co/Soofi-Project/Soofi-S-Base#license-terms-of-use", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2607.09424", "title": null } ], "concepts": [ { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-soofi-s-30b-a3b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/soofi-s-30b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/soofi-s-30b-a3b.md" }, { "slug": "laguna-xs-2-1", "order": 84, "name": "Laguna XS 2.1 (33B)", "baseTitle": "Laguna XS 2.1", "company": "Poolside", "country": "United States", "flag": "🇺🇸", "date": "2026-07-02", "decoderType": "Sparse MoE", "attention": "Gated GQA with QK-Norm and 3:1 sliding-window/global attention", "layerMix": "30 sliding-window + 10 global GQA", "scale": "33B total, 3B active (9.1% active)", "totalParams": 33000000000, "activeParams": 3000000000, "context": "262,144", "contextTokens": 262144, "license": "OpenMDW License 1.1", "kvCacheText": "160 KiB", "kvCacheKiB": 160, "kvCacheLevel": "Moderate", "summary": "Poolside's 33B Laguna refresh extends the sparse MoE architecture to 262k context while keeping about 3B parameters active per token.", "keyDetail": "Uses 256 routed experts with top-8 routing plus one shared expert, a dense first layer, and per-layer query-head counts.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/laguna-xs-2-1.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/laguna-xs-2-1.webp", "alt": "Figure 1: Laguna XS 2.1", "width": 960, "height": 913 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/main/config.json", "title": "poolside/Laguna-XS-2.1" }, { "label": "License", "href": "https://huggingface.co/poolside/Laguna-XS-2.1/blob/main/LICENSE.md", "title": null }, { "label": "Tech report", "href": "https://huggingface.co/poolside/Laguna-XS-2.1", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "attention-budgeting", "label": "Attention Budgeting", "name": "Layer-wise attention budgeting", "url": "https://llm-atlas-7or.pages.dev/concepts/attention-budgeting/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-laguna-xs-2-1", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/laguna-xs-2-1/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/laguna-xs-2-1.md" }, { "slug": "glm-5-2", "order": 83, "name": "GLM-5.2 (744B)", "baseTitle": "GLM-5.2", "company": "Z.ai", "country": "China", "flag": "🇨🇳", "date": "2026-06-17", "decoderType": "Sparse MoE", "attention": "MLA with DeepSeek Sparse Attention and IndexShare", "layerMix": "78 MLA/DSA", "scale": "744B total, 40B active (5.4% active)", "totalParams": 744000000000, "activeParams": 40000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "87.8 KiB", "kvCacheKiB": 87.8, "kvCacheLevel": "Moderate", "summary": "Long-context GLM refresh that keeps the 744B sparse MoE backbone and adds IndexShare for 1M-token DSA inference.", "keyDetail": "Reuses each full DSA indexer result across the next three layers, making 1M-token sparse attention cheaper.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/glm-5.2.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/glm-5.2.webp", "alt": "Figure 1: GLM-5.2", "width": 960, "height": 934 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/zai-org/GLM-5.2/blob/main/config.json", "title": "zai-org/GLM-5.2" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2602.15763", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "indexshare", "label": "IndexShare", "name": "IndexShare", "url": "https://llm-atlas-7or.pages.dev/concepts/indexshare/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 33.7, "profile": { "General": 36.9, "Scientific": 31, "Coding": 26.1, "Agents": 37.5 }, "url": "https://artificialanalysis.ai/models/glm-5-2" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-glm-5-2", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/glm-5-2/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/glm-5-2.md" }, { "slug": "vibethinker-3b", "order": 82, "name": "VibeThinker-3B (3B)", "baseTitle": "VibeThinker-3B", "company": "Weibo", "country": "China", "flag": "🇨🇳", "date": "2026-06-15", "decoderType": "Dense", "attention": "8:1 GQA with RoPE", "layerMix": "36 GQA", "scale": "3B parameters", "totalParams": 3000000000, "activeParams": 3000000000, "context": "131,072", "contextTokens": 131072, "license": "MIT License", "kvCacheText": "36 KiB", "kvCacheKiB": 36, "kvCacheLevel": "Low", "summary": "Qwen2.5-Coder-based dense reasoner whose gains mainly come from verifier-driven post-training rather than a new block design.", "keyDetail": "Standard Qwen2.5-Coder-style dense stack strengthened by curriculum SFT, MGPO RL, Long2Short RL, offline self-distillation, and Instruct RL.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/vibethinker-3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/vibethinker-3b.webp", "alt": "Figure 1: VibeThinker-3B", "width": 921, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/WeiboAI/VibeThinker-3B/blob/main/config.json", "title": "WeiboAI/VibeThinker-3B" }, { "label": "License", "href": "https://huggingface.co/WeiboAI/VibeThinker-3B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2606.16140", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-vibethinker-3b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/vibethinker-3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/vibethinker-3b.md" }, { "slug": "minimax-m3-428b", "order": 81, "name": "MiniMax M3 (428B)", "baseTitle": "MiniMax M3 428B", "company": "MiniMax", "country": "China", "flag": "🇨🇳", "date": "2026-06-13", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm and MiniMax Sparse Attention", "layerMix": "3 full GQA + 57 MiniMax Sparse Attention", "scale": "428B total, 23B active (5.4% active)", "totalParams": 428000000000, "activeParams": 23000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MiniMax Community License", "kvCacheText": "120 KiB", "kvCacheKiB": 120, "kvCacheLevel": "Moderate", "summary": "Native multimodal MiniMax model that adds sparse attention to an M2-style MoE backbone for 1M-token context.", "keyDetail": "Uses three dense prefix layers, then 57 sparse-attention MoE layers with 128 routed experts and 4 routed plus 1 shared expert active per token.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/minimax-m3.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/minimax-m3.webp", "alt": "Figure 1: MiniMax M3 428B", "width": 960, "height": 941 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/MiniMaxAI/MiniMax-M3/blob/main/config.json", "title": "MiniMaxAI/MiniMax-M3" }, { "label": "License", "href": "https://huggingface.co/MiniMaxAI/MiniMax-M3/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2606.13392", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 18.7, "profile": { "Knowledge Work": 23.9, "Coding": 13.9, "Science": 10.9, "General": 26.1 }, "url": "https://artificialanalysis.ai/models/minimax-m3" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 29.2, "profile": { "General": 36.3, "Scientific": 21.3, "Coding": 24.6, "Agents": 30.7 }, "url": "https://artificialanalysis.ai/models/minimax-m3" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-minimax-m3-428b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/minimax-m3-428b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/minimax-m3-428b.md" }, { "slug": "kimi-k2-7-code", "order": 80, "name": "Kimi K2.7 Code (1T)", "baseTitle": "Kimi K2.7 Code", "company": "Moonshot AI", "country": "China", "flag": "🇨🇳", "date": "2026-06-12", "decoderType": "Sparse MoE", "attention": "MLA", "layerMix": "61 MLA", "scale": "1T total, 32B active (3.2% active)", "totalParams": 1000000000000, "activeParams": 32000000000, "context": "256,000", "contextTokens": 256000, "license": "Modified MIT License", "kvCacheText": "68.6 KiB", "kvCacheKiB": 68.6, "kvCacheLevel": "Low", "summary": "Coding-focused K2.6 successor that keeps the same 1T sparse MoE text backbone and emphasizes long-horizon software engineering agents.", "keyDetail": "Keeps the Kimi K2.5/K2.6 text architecture and shifts the release focus toward coding-agent workflows.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/kimi-k2-7-code.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/kimi-k2-7-code.webp", "alt": "Figure 1: Kimi K2.7 Code", "width": 960, "height": 930 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/moonshotai/Kimi-K2.7-Code/blob/main/config.json", "title": "moonshotai/Kimi-K2.7-Code" }, { "label": "License", "href": "https://huggingface.co/moonshotai/Kimi-K2.7-Code/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://huggingface.co/moonshotai/Kimi-K2.7-Code", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 25.8, "profile": { "General": 33.1, "Scientific": 22.5, "Coding": 24.4, "Agents": 22 }, "url": "https://artificialanalysis.ai/models/kimi-k2-7-code" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-kimi-k2-7-code", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/kimi-k2-7-code/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/kimi-k2-7-code.md" }, { "slug": "north-mini-code-30b-a3b", "order": 79, "name": "North Mini Code (30B-A3B)", "baseTitle": "North Mini Code 30B-A3B", "company": "Cohere", "country": "Canada", "flag": "🇨🇦", "date": "2026-06-05", "decoderType": "Sparse MoE", "attention": "8:1 GQA with 3:1 sliding-window/global attention", "layerMix": "36 sliding-window + 13 global GQA/MoE", "scale": "30B total, 3B active (10% active)", "totalParams": 30000000000, "activeParams": 3000000000, "context": "256K input, 64K output", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "98 KiB", "kvCacheKiB": 98, "kvCacheLevel": "Moderate", "summary": "Cohere's compact agentic coding MoE adapts the parallel Cohere2-MoE block to a 30B total, 3B active model.", "keyDetail": "Runs attention and MoE in parallel, with RoPE in sliding-window layers, NoPE in global layers, and top-8 routing over 128 experts.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/north-mini-code.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/north-mini-code.webp", "alt": "Figure 1: North Mini Code 30B-A3B", "width": 960, "height": 869 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0/blob/main/config.json", "title": "CohereLabs/North-Mini-Code-1.0" }, { "label": "Tech report", "href": "https://huggingface.co/blog/CohereLabs/introducing-north-mini-code", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 9.9, "profile": { "General": 15.9, "Scientific": 5.7, "Coding": 19.6, "Agents": 0.2 }, "url": "https://artificialanalysis.ai/models/north-mini-code" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-north-mini-code-30b-a3b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/north-mini-code-30b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/north-mini-code-30b-a3b.md" }, { "slug": "nemotron-3-ultra-550b-a55b", "order": 78, "name": "Nemotron 3 Ultra (550B-A55B)", "baseTitle": "Nemotron 3 Ultra 550B-A55B", "company": "NVIDIA", "country": "United States", "flag": "🇺🇸", "date": "2026-06-04", "decoderType": "Hybrid MoE", "attention": "Mostly Mamba-2 with a few GQA layers", "layerMix": "12 GQA + 48 Mamba-2 + 48 MoE", "scale": "550B total, 55B active (10% active)", "totalParams": 550000000000, "activeParams": 55000000000, "context": "1,000,000", "contextTokens": 1000000, "license": "NVIDIA Nemotron Open Model License", "kvCacheText": "12 KiB", "kvCacheKiB": 12, "kvCacheLevel": "Very low", "summary": "NVIDIA's Ultra model scales the Nemotron 3 hybrid stack to 550B total parameters while keeping 55B active per token.", "keyDetail": "Scales the latent-MoE hybrid to 108 layers with 48 Mamba-2, 48 latent MoE, and 12 GQA layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/nemotron-3-ultra-550b-a55b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/nemotron-3-ultra-550b-a55b.webp", "alt": "Figure 1: Nemotron 3 Ultra 550B-A55B", "width": 960, "height": 931 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4/blob/main/config.json", "title": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4" }, { "label": "License", "href": "https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-nemotron-open-model-license/", "title": null }, { "label": "Tech report", "href": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Ultra-Technical-Report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 11.8, "profile": { "Knowledge Work": 12.9, "Coding": 0.9, "Science": 6, "General": 27.4 }, "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-ultra-550b-a55b" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 22.9, "profile": { "General": 34.1, "Scientific": 15.8, "Coding": 20.4, "Agents": 18.5 }, "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-ultra-550b-a55b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-nemotron-3-ultra-550b-a55b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/nemotron-3-ultra-550b-a55b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/nemotron-3-ultra-550b-a55b.md" }, { "slug": "gemma-4-12b", "order": 37, "name": "Gemma 4 (12B)", "baseTitle": "Gemma 4 12B", "company": "Google", "country": "United States", "flag": "🇺🇸", "date": "2026-06-03", "decoderType": "Dense", "attention": "GQA with QK-Norm, unified K/V on global layers, p-RoPE on global layers, and 5:1 sliding-window/global attention", "layerMix": "40 sliding-window + 8 global", "scale": "12B parameters", "totalParams": 12000000000, "activeParams": 12000000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "328 KiB", "kvCacheKiB": 328, "kvCacheLevel": "Very high", "summary": "Dense Gemma 4 12B Unified uses an encoder-free multimodal input path, while its text decoder keeps the 31B-style local-global attention recipe.", "keyDetail": "Figure shows the 48-layer text decoder; the full 12B Unified model also projects audio, image, and video inputs directly into the decoder input stream.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gemma-4-12b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gemma-4-12b.webp", "alt": "Figure 59: Gemma 4 12B", "width": 918, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/google/gemma-4-12B-it/blob/main/config.json", "title": "google/gemma-4-12B-it" }, { "label": "Tech report", "href": "https://developers.googleblog.com/gemma-4-12b-the-developer-guide/", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 9.4, "profile": { "Scientific": 3.2 }, "url": "https://artificialanalysis.ai/models/gemma-4-12b-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gemma-4-12b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/gemma-4-12b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gemma-4-12b.md" }, { "slug": "jetbrains-mellum2-thinking-12b-a2-5b", "order": 77, "name": "JetBrains Mellum2 Thinking (12B-A2.5B)", "baseTitle": "JetBrains Mellum2 Thinking 12B-A2.5B", "company": "JetBrains", "country": "Czech Republic", "flag": "🇨🇿", "date": "2026-06-01", "decoderType": "Sparse MoE", "attention": "GQA with 3:1 sliding-window/full attention", "layerMix": "21 sliding-window + 7 full GQA", "scale": "12B total, 2.5B active (20.8% active)", "totalParams": 12000000000, "activeParams": 2500000000, "context": "131,072", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "56 KiB", "kvCacheKiB": 56, "kvCacheLevel": "Low", "summary": "JetBrains' Mellum2 Thinking is a compact reasoning-tuned sparse MoE for code and agentic workflows.", "keyDetail": "Uses 64 experts with 8 active per token, a 1024-token sliding window, and YaRN scaling on full-attention layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/mellum2.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/mellum2.webp", "alt": "Figure 1: JetBrains Mellum2 Thinking 12B-A2.5B", "width": 960, "height": 925 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/JetBrains/Mellum2-12B-A2.5B-Thinking/blob/main/config.json", "title": "JetBrains/Mellum2-12B-A2.5B-Thinking" }, { "label": "Tech report", "href": "https://arxiv.org/abs/2605.31268", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-jetbrains-mellum2-thinking-12b-a2-5b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/jetbrains-mellum2-thinking-12b-a2-5b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/jetbrains-mellum2-thinking-12b-a2-5b.md" }, { "slug": "lfm2-5-1-2b", "order": 74, "name": "LFM2.5 (1.2B)", "baseTitle": "LFM2.5 1.2B", "company": "Liquid AI", "country": "United States", "flag": "🇺🇸", "date": "2026-05-28", "decoderType": "Dense hybrid", "attention": "LIV convolution blocks plus GQA", "layerMix": "10 LIV conv + 6 GQA", "scale": "1.2B parameters", "totalParams": 1200000000, "activeParams": 1200000000, "context": "128,000", "contextTokens": 128000, "license": "LFM Open License v1.0", "kvCacheText": "12 KiB", "kvCacheKiB": 12, "kvCacheLevel": "Very low", "summary": "Liquid AI's 1.2B LFM2.5 model keeps the 16-layer hybrid LIV convolution and GQA stack at a wider hidden size.", "keyDetail": "Keeps the 16-layer LFM2.5 hybrid stack but doubles the hidden size to 2048 and uses 32 query heads.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/lfm2-5-1-2b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/lfm2-5-1-2b.webp", "alt": "Figure 1: LFM2.5 1.2B", "width": 830, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/main/config.json", "title": "LiquidAI/LFM2.5-1.2B-Thinking" }, { "label": "License", "href": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Thinking/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://www.liquid.ai/blog/lfm2-5-1-2b-thinking-on-device-reasoning-under-1gb", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 5.2, "profile": { "Scientific": 3.1 }, "url": "https://artificialanalysis.ai/models/lfm2-5-1-2b-thinking" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-lfm2-5-1-2b", "family": "hybrid", "url": "https://llm-atlas-7or.pages.dev/models/lfm2-5-1-2b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/lfm2-5-1-2b.md" }, { "slug": "lfm2-5-350m", "order": 75, "name": "LFM2.5 (350M)", "baseTitle": "LFM2.5 350M", "company": "Liquid AI", "country": "United States", "flag": "🇺🇸", "date": "2026-05-28", "decoderType": "Dense hybrid", "attention": "LIV convolution blocks plus GQA", "layerMix": "10 LIV conv + 6 GQA", "scale": "350M parameters", "totalParams": 350000000, "activeParams": 350000000, "context": "128,000", "contextTokens": 128000, "license": "LFM Open License v1.0", "kvCacheText": "12 KiB", "kvCacheKiB": 12, "kvCacheLevel": "Very low", "summary": "Liquid AI's compact LFM2.5 model combines double-gated LIV convolution blocks with periodic GQA layers.", "keyDetail": "Uses a 16-layer hybrid stack with 10 LIV convolution layers and 6 GQA layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/lfm2-5-350m.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/lfm2-5-350m.webp", "alt": "Figure 1: LFM2.5 350M", "width": 830, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/LiquidAI/LFM2.5-350M/blob/main/config.json", "title": "LiquidAI/LFM2.5-350M" }, { "label": "License", "href": "https://huggingface.co/LiquidAI/LFM2.5-350M/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://www.liquid.ai/blog/lfm2-5-350m-no-size-left-behind", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-lfm2-5-350m", "family": "hybrid", "url": "https://llm-atlas-7or.pages.dev/models/lfm2-5-350m/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/lfm2-5-350m.md" }, { "slug": "lfm2-5-8b-a1b", "order": 76, "name": "LFM2.5 (8B-A1B)", "baseTitle": "LFM2.5 8B-A1B", "company": "Liquid AI", "country": "United States", "flag": "🇺🇸", "date": "2026-05-28", "decoderType": "Sparse hybrid", "attention": "LIV convolution blocks plus GQA and MoE", "layerMix": "18 LIV conv + 6 GQA + MoE FFNs", "scale": "8.3B total, 1.5B active (18.1% active)", "totalParams": 8300000000.000001, "activeParams": 1500000000, "context": "128,000", "contextTokens": 128000, "license": "LFM Open License v1.0", "kvCacheText": "12 KiB", "kvCacheKiB": 12, "kvCacheLevel": "Very low", "summary": "Liquid AI's sparse LFM2.5 model combines LIV convolution, GQA, and MoE routing with 4 of 32 experts active per token.", "keyDetail": "Uses 24 layers, 32 MoE experts with 4 active per token, and leaves the first two layers dense before sparse routing begins.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/lfm2-5-8b-a1b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/lfm2-5-8b-a1b.webp", "alt": "Figure 1: LFM2.5 8B-A1B", "width": 856, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/main/config.json", "title": "LiquidAI/LFM2.5-8B-A1B" }, { "label": "License", "href": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://www.liquid.ai/blog/lfm2-5-8b-a1b", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 7.2, "profile": { "Scientific": 3.4 }, "url": "https://artificialanalysis.ai/models/lfm2-5-8b-a1b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-lfm2-5-8b-a1b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/lfm2-5-8b-a1b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/lfm2-5-8b-a1b.md" }, { "slug": "command-a-218b-a25b", "order": 73, "name": "Command A+ (218B-A25B)", "baseTitle": "Command A+ 218B-A25B", "company": "Cohere", "country": "Canada", "flag": "🇨🇦", "date": "2026-05-20", "decoderType": "Sparse MoE", "attention": "16:1 GQA with 3:1 sliding-window/global attention", "layerMix": "24 sliding-window + 8 global GQA/MoE", "scale": "218B total, 25B active/token (11.5% active)", "totalParams": 218000000000, "activeParams": 25000000000, "context": "128K input, 64K output", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "128 KiB", "kvCacheKiB": 128, "kvCacheLevel": "Moderate", "summary": "Cohere's open sparse MoE model scales the Command A parallel transformer block to 218B total parameters while keeping 25B active per token.", "keyDetail": "Runs attention and MoE in parallel, which Cohere reports gives similar quality to a sequential block with better throughput.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/command-a-plus.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/command-a-plus.webp", "alt": "Figure 1: Command A+ 218B-A25B", "width": 960, "height": 939 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/CohereLabs/command-a-plus-05-2026-w4a4/blob/main/config.json", "title": "CohereLabs/command-a-plus-05-2026-w4a4" }, { "label": "Tech report", "href": "https://cohere.com/blog/command-a-plus", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 6.2, "profile": { "Knowledge Work": 0.6, "Coding": 0.2, "Science": 1, "General": 23.2 }, "url": "https://artificialanalysis.ai/models/command-a-plus" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 13.1, "profile": { "General": 26.4, "Scientific": 6.1, "Coding": 19.5, "Agents": 0.5 }, "url": "https://artificialanalysis.ai/models/command-a-plus" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-command-a-218b-a25b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/command-a-218b-a25b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/command-a-218b-a25b.md" }, { "slug": "zaya1-8b", "order": 63, "name": "ZAYA1-8B (8.4B)", "baseTitle": "ZAYA1-8B", "company": "Zyphra", "country": "United States", "flag": "🇺🇸", "date": "2026-05-06", "decoderType": "Sparse MoE", "attention": "CCA with 4:1 GQA, RoPE, and Q/K L2 norm", "layerMix": "40 CCA/GQA attention + 40 top-1 MoE FFN entries", "scale": "8.4B total, 760M active/token (9% active)", "totalParams": 8400000000, "activeParams": 760000000, "context": "131,072", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "40 KiB", "kvCacheKiB": 40, "kvCacheLevel": "Low", "summary": "Zyphra's compact reasoning MoE pairs CCA/GQA attention updates with top-1 routed expert FFN updates to keep fewer than 1B parameters active per token.", "keyDetail": "Key detail is CCA/GQA attention, which performs attention in a compressed latent space while keeping a 4:1 GQA layout. Sidenote: the config lists 80 alternating layer entries, equivalent to 40 CCA/GQA attention updates paired with 40 top-1 MoE FFN updates.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/zaya1-8b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/zaya1-8b.webp", "alt": "Figure 11: ZAYA1-8B", "width": 960, "height": 958 }, "articleUrl": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A74-compressed-convolutional-attention-zaya1-8b", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A74-compressed-convolutional-attention-zaya1-8b", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Zyphra/ZAYA1-8B/blob/main/config.json", "title": "Zyphra/ZAYA1-8B" }, { "label": "Tech report", "href": "https://arxiv.org/abs/2605.05365", "title": null } ], "concepts": [ { "slug": "compressed-convolutional-attention", "label": "CCA", "name": "Compressed Convolutional Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/compressed-convolutional-attention/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-zaya1-8b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/zaya1-8b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/zaya1-8b.md" }, { "slug": "granite-4-1-30b", "order": 72, "name": "Granite 4.1 (30B)", "baseTitle": "Granite 4.1 30B", "company": "IBM", "country": "United States", "flag": "🇺🇸", "date": "2026-04-29", "decoderType": "Dense", "attention": "GQA with RoPE", "layerMix": "64 GQA", "scale": "30B parameters", "totalParams": 30000000000, "activeParams": 30000000000, "context": "131,072", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "256 KiB", "kvCacheKiB": 256, "kvCacheLevel": "High", "summary": "IBM's dense 30B Granite 4.1 model keeps a straightforward long-context GQA transformer stack and improves tool calling, instruction following, and chat behavior through post-training.", "keyDetail": "Uses a 4096-wide dense stack, 32 query heads, 8 KV heads, SwiGLU MLPs, shared input/output embeddings, and Granite-specific MuP scaling factors.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/granite-4.1.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/granite-4.1.webp", "alt": "Figure 1: Granite 4.1 30B", "width": 918, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/ibm-granite/granite-4.1-30b/blob/main/config.json", "title": "ibm-granite/granite-4.1-30b" }, { "label": "Tech report", "href": "https://huggingface.co/blog/ibm-granite/granite-4-1", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 7.4, "profile": { "Scientific": 2.1 }, "url": "https://artificialanalysis.ai/models/granite-4-1-30b/" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-granite-4-1-30b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/granite-4-1-30b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/granite-4-1-30b.md" }, { "slug": "laguna-xs-2", "order": 62, "name": "Laguna XS.2 (33B)", "baseTitle": "Laguna XS.2", "company": "Poolside", "country": "United States", "flag": "🇺🇸", "date": "2026-04-28", "decoderType": "Sparse MoE", "attention": "Gated GQA with QK-Norm and 3:1 sliding-window/global attention", "layerMix": "30 sliding-window + 10 global", "scale": "33B total, 3B active (9.1% active)", "totalParams": 33000000000, "activeParams": 3000000000, "context": "131,072", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "160 KiB", "kvCacheKiB": 160, "kvCacheLevel": "Moderate", "summary": "Poolside's open-weight agentic coding MoE uses mixed sliding-window/global attention and activates only 3B of 33B parameters per token.", "keyDetail": "Uses per-layer query-head counts, a 512-token SWA window, sigmoid MoE routing, and one shared expert alongside the top-8 routed experts.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/laguna-xs2.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/laguna-xs2.webp", "alt": "Figure 9: Laguna XS.2", "width": 960, "height": 913 }, "articleUrl": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A73-layer-wise-attention-budgeting-laguna-xs2", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A73-layer-wise-attention-budgeting-laguna-xs2", "title": null }, { "label": "config.json", "href": "https://huggingface.co/poolside/Laguna-XS.2/blob/main/config.json", "title": "poolside/Laguna-XS.2" }, { "label": "License", "href": "https://huggingface.co/poolside/Laguna-XS.2/blob/main/LICENSE.md", "title": null }, { "label": "Tech report", "href": "https://poolside.ai/assets/laguna/laguna-m1-xs2-technical-report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "attention-budgeting", "label": "Attention Budgeting", "name": "Layer-wise attention budgeting", "url": "https://llm-atlas-7or.pages.dev/concepts/attention-budgeting/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-laguna-xs-2", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/laguna-xs-2/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/laguna-xs-2.md" }, { "slug": "deepseek-v4-flash", "order": 60, "name": "DeepSeek V4-Flash (284B)", "baseTitle": "DeepSeek V4-Flash", "company": "DeepSeek", "country": "China", "flag": "🇨🇳", "date": "2026-04-24", "decoderType": "Sparse MoE", "attention": "MLA-style CSA/HCA with mHC", "layerMix": "43 CSA/HCA", "scale": "284B total, 13B active (4.6% active)", "totalParams": 284000000000, "activeParams": 13000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "5.4 KiB", "kvCacheKiB": 5.4, "kvCacheLevel": "Very low", "summary": "DeepSeek's efficient V4 preview keeps the million-token architecture while reducing the MoE scale to 284B parameters and 13B active parameters.", "keyDetail": "Uses 256 experts, 6 routed plus 1 shared expert per token, hash-based routing, compressed attention, and the V4 MTP path.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/deepseek-v4-flash.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/deepseek-v4-flash.webp", "alt": "Figure 17: DeepSeek V4-Flash", "width": 960, "height": 774 }, "articleUrl": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A75-csahca-mhc-and-compressed-attention-caches-deepseek-v4", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A75-csahca-mhc-and-compressed-attention-caches-deepseek-v4", "title": null }, { "label": "config.json", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/config.json", "title": "deepseek-ai/DeepSeek-V4-Flash" }, { "label": "License", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2606.19348", "title": null } ], "concepts": [ { "slug": "csa-hca", "label": "CSA/HCA", "name": "Compressed Sparse & Heavily Compressed Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/csa-hca/" }, { "slug": "mhc", "label": "mHC", "name": "Manifold-constrained Hyper-Connections", "url": "https://llm-atlas-7or.pages.dev/concepts/mhc/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 34.3, "profile": { "General": 31.8, "Scientific": 27.6, "Coding": 31.2, "Agents": 43.5 }, "url": "https://artificialanalysis.ai/models/deepseek-v4-flash" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-deepseek-v4-flash", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/deepseek-v4-flash/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/deepseek-v4-flash.md" }, { "slug": "deepseek-v4-pro", "order": 61, "name": "DeepSeek V4-Pro (1.6T)", "baseTitle": "DeepSeek V4-Pro", "company": "DeepSeek", "country": "China", "flag": "🇨🇳", "date": "2026-04-24", "decoderType": "Sparse MoE", "attention": "MLA-style CSA/HCA with mHC", "layerMix": "61 CSA/HCA", "scale": "1.6T total, 49B active (3.1% active)", "totalParams": 1600000000000, "activeParams": 49000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "7.7 KiB", "kvCacheKiB": 7.7, "kvCacheLevel": "Very low", "summary": "DeepSeek's flagship V4 preview scales to 1.6T parameters and introduces compressed sparse attention plus manifold-constrained hyper-connections for million-token contexts.", "keyDetail": "Uses 384 experts, 6 routed plus 1 shared expert per token, hash-based routing, compressed attention caches, and the V4 MTP path.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/deepseek-v4-pro.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/deepseek-v4-pro.webp", "alt": "Figure 17: DeepSeek V4-Pro", "width": 960, "height": 774 }, "articleUrl": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A75-csahca-mhc-and-compressed-attention-caches-deepseek-v4", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A75-csahca-mhc-and-compressed-attention-caches-deepseek-v4", "title": null }, { "label": "config.json", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/config.json", "title": "deepseek-ai/DeepSeek-V4-Pro" }, { "label": "License", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2606.19348", "title": null } ], "concepts": [ { "slug": "csa-hca", "label": "CSA/HCA", "name": "Compressed Sparse & Heavily Compressed Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/csa-hca/" }, { "slug": "mhc", "label": "mHC", "name": "Manifold-constrained Hyper-Connections", "url": "https://llm-atlas-7or.pages.dev/concepts/mhc/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 23.1, "profile": { "Knowledge Work": 36.3, "Coding": 15.9, "Science": 12.8, "General": 27.1 }, "url": "https://artificialanalysis.ai/models/deepseek-v4-pro" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 36, "profile": { "General": 34.4, "Scientific": 29.5, "Coding": 32.6, "Agents": 44.2 }, "url": "https://artificialanalysis.ai/models/deepseek-v4-pro" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-deepseek-v4-pro", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/deepseek-v4-pro/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/deepseek-v4-pro.md" }, { "slug": "ling-2-6-1t", "order": 70, "name": "Ling 2.6 (1T)", "baseTitle": "Ling 2.6 1T", "company": "InclusionAI", "country": "China", "flag": "🇨🇳", "date": "2026-04-23", "decoderType": "Sparse hybrid", "attention": "Lightning Attention plus MLA", "layerMix": "10 MLA + 70 Lightning Attention + 1 MTP layer", "scale": "1T total, 63B active (6.3% active)", "totalParams": 1000000000000, "activeParams": 63000000000, "context": "262,144", "contextTokens": 262144, "license": "MIT License", "kvCacheText": "11.2 KiB", "kvCacheKiB": 11.2, "kvCacheLevel": "Very low", "summary": "Ling 2.5 successor that keeps the Lightning Attention and MLA hybrid stack while adding an MTP layer for multi-token prediction.", "keyDetail": "Keeps the 7:1 linear-attention/MLA ratio and adds one multi-token-prediction layer for speculative decoding.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/ling-2-6-1t.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/ling-2-6-1t.webp", "alt": "Figure 1: Ling 2.6 1T", "width": 960, "height": 943 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/inclusionAI/Ling-2.6-1T/blob/main/config.json", "title": "inclusionAI/Ling-2.6-1T" }, { "label": "Tech report", "href": "https://huggingface.co/inclusionAI/Ling-2.6-1T", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 17, "profile": { "Scientific": 4.5 }, "url": "https://artificialanalysis.ai/models/ling-2-6-1t" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-ling-2-6-1t", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/ling-2-6-1t/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/ling-2-6-1t.md" }, { "slug": "tencent-hy3-preview-295b-a21b", "order": 71, "name": "Tencent Hy3-preview (295B-A21B)", "baseTitle": "Tencent Hy3-preview 295B-A21B", "company": "Tencent", "country": "China", "flag": "🇨🇳", "date": "2026-04-23", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm", "layerMix": "80 GQA + 1 MTP layer", "scale": "295B total, 21B active (7.1% active)", "totalParams": 295000000000, "activeParams": 21000000000, "context": "262,144", "contextTokens": 262144, "license": "Tencent Hy Community License Agreement", "kvCacheText": "320 KiB", "kvCacheKiB": 320, "kvCacheLevel": "Very high", "summary": "Tencent Hunyuan's 295B sparse MoE preview combines a dense first FFN layer, GQA with QK-Norm, and one extra MTP layer for faster generation.", "keyDetail": "Uses 192 routed experts, 8 routed plus 1 shared expert active per token, sigmoid routing with expert bias, and a 3.8B MTP layer.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/tencent-hy3.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/tencent-hy3.webp", "alt": "Figure 1: Tencent Hy3-preview 295B-A21B", "width": 960, "height": 904 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/tencent/Hy3-preview/blob/main/config.json", "title": "tencent/Hy3-preview" }, { "label": "License", "href": "https://huggingface.co/tencent/Hy3-preview/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://www.tencent.com/en-us/articles/2202320.html", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 17, "profile": { "Scientific": 3.6 }, "url": "https://artificialanalysis.ai/models/hy3-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-tencent-hy3-preview-295b-a21b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/tencent-hy3-preview-295b-a21b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/tencent-hy3-preview-295b-a21b.md" }, { "slug": "qwen3-6-27b", "order": 67, "name": "Qwen3.6 (27B)", "baseTitle": "Qwen3.6 27B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2026-04-22", "decoderType": "Dense hybrid", "attention": "3:1 Gated DeltaNet and Gated Attention", "layerMix": "16 gated attention + 48 DeltaNet", "scale": "27B parameters", "totalParams": 27000000000, "activeParams": 27000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "64 KiB", "kvCacheKiB": 64, "kvCacheLevel": "Low", "summary": "Dense Qwen3.6 model that keeps the Qwen3.5-style Gated DeltaNet/Gated Attention hybrid stack while replacing MoE blocks with dense FFNs.", "keyDetail": "Uses a 64-layer dense hybrid layout with 48 DeltaNet layers and 16 full-attention layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-6-27b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-6-27b.webp", "alt": "Figure 1: Qwen3.6 27B", "width": 960, "height": 904 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/main/config.json", "title": "Qwen/Qwen3.6-27B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3.6-27B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://qwen.ai/blog?id=qwen3.6-27b", "title": null } ], "concepts": [ { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 21.4, "profile": { "General": 31.5, "Scientific": 12.1, "Coding": 21.4, "Agents": 17.8 }, "url": "https://artificialanalysis.ai/models/qwen3-6-27b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-6-27b", "family": "hybrid", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-6-27b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-6-27b.md" }, { "slug": "xiaomi-mimo-v2-5-310b", "order": 68, "name": "Xiaomi MiMo-V2.5 (310B)", "baseTitle": "Xiaomi MiMo-V2.5 310B", "company": "Xiaomi", "country": "China", "flag": "🇨🇳", "date": "2026-04-22", "decoderType": "Sparse omnimodal MoE", "attention": "5:1 sliding-window/global attention", "layerMix": "39 sliding-window + 9 global", "scale": "310B total, 15B active (4.8% active)", "totalParams": 310000000000, "activeParams": 15000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "144 KiB", "kvCacheKiB": 144, "kvCacheLevel": "Moderate", "summary": "Omnimodal sparse MoE model that extends the MiMo-V2-Flash backbone with vision and audio encoders.", "keyDetail": "Adds native image, video, and audio encoders to the MiMo-V2-Flash-style sparse MoE backbone with MTP support.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/xiaomi-mimo-v2-5-310b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/xiaomi-mimo-v2-5-310b.webp", "alt": "Figure 1: Xiaomi MiMo-V2.5 310B", "width": 960, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5/blob/main/config.json", "title": "XiaomiMiMo/MiMo-V2.5" }, { "label": "Tech report", "href": "https://mimo.xiaomi.com/mimo-v2-5", "title": null } ], "concepts": [ { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 21.8, "profile": { "General": 30.4, "Scientific": 15.5, "Coding": 21.9, "Agents": 17.7 }, "url": "https://artificialanalysis.ai/models/mimo-v2-5-0424" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-xiaomi-mimo-v2-5-310b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-5-310b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-5-310b.md" }, { "slug": "xiaomi-mimo-v2-5-pro-1-02t", "order": 69, "name": "Xiaomi MiMo-V2.5-Pro (1.02T)", "baseTitle": "Xiaomi MiMo-V2.5-Pro 1.02T", "company": "Xiaomi", "country": "China", "flag": "🇨🇳", "date": "2026-04-22", "decoderType": "Sparse MoE", "attention": "GQA with 6:1 sliding-window/global attention", "layerMix": "60 sliding-window + 10 global", "scale": "1.02T total, 42B active (4.1% active)", "totalParams": 1020000000000, "activeParams": 42000000000, "context": "1,048,576", "contextTokens": 1048576, "license": "MIT License", "kvCacheText": "350 KiB", "kvCacheKiB": 350, "kvCacheLevel": "Very high", "summary": "Trillion-parameter sparse MoE model for long-horizon agentic and software-engineering tasks.", "keyDetail": "Scales the MiMo-V2.5 Pro stack to 70 layers, 384 routed experts, top-8 routing, and a 1M-token context window.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/xiaomi-mimo25-pro.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/xiaomi-mimo25-pro.webp", "alt": "Figure 1: Xiaomi MiMo-V2.5-Pro 1.02T", "width": 960, "height": 938 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/blob/main/config.json", "title": "XiaomiMiMo/MiMo-V2.5-Pro" }, { "label": "Tech report", "href": "https://mimo.xiaomi.com/mimo-v2-5", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 26, "profile": { "General": 34.6, "Scientific": 19.8, "Coding": 25.3, "Agents": 22.2 }, "url": "https://artificialanalysis.ai/models/mimo-v2-5-pro" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-xiaomi-mimo-v2-5-pro-1-02t", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-5-pro-1-02t/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-5-pro-1-02t.md" }, { "slug": "kimi-k2-6", "order": 66, "name": "Kimi K2.6 (1T)", "baseTitle": "Kimi K2.6", "company": "Moonshot AI", "country": "China", "flag": "🇨🇳", "date": "2026-04-20", "decoderType": "Sparse MoE", "attention": "MLA", "layerMix": "61 MLA", "scale": "1T total, 32B active (3.2% active)", "totalParams": 1000000000000, "activeParams": 32000000000, "context": "256,000", "contextTokens": 256000, "license": "Modified MIT License", "kvCacheText": "68.6 KiB", "kvCacheKiB": 68.6, "kvCacheLevel": "Low", "summary": "Native-multimodal K2.5 successor that keeps the same 1T sparse MoE backbone while targeting stronger long-horizon coding, design, and agent orchestration.", "keyDetail": "Uses the same text architecture as Kimi K2.5, with the main change coming from the multimodal and agentic training recipe.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/kimi-k2-6.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/kimi-k2-6.webp", "alt": "Figure 1: Kimi K2.6", "width": 960, "height": 930 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/moonshotai/Kimi-K2.6/blob/main/config.json", "title": "moonshotai/Kimi-K2.6" }, { "label": "License", "href": "https://huggingface.co/moonshotai/Kimi-K2.6/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://www.kimi.com/blog/kimi-k2-6.html", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 27, "profile": { "General": 38.6, "Scientific": 22.7, "Coding": 26, "Agents": 19.1 }, "url": "https://artificialanalysis.ai/models/kimi-k2-6" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-kimi-k2-6", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/kimi-k2-6/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/kimi-k2-6.md" }, { "slug": "qwen3-6-35b-a3b", "order": 65, "name": "Qwen3.6 (35B-A3B)", "baseTitle": "Qwen3.6 35B-A3B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2026-04-15", "decoderType": "Sparse hybrid", "attention": "3:1 Gated DeltaNet and Gated Attention", "layerMix": "10 gated attention + 30 DeltaNet", "scale": "35B total, 3B active (8.6% active)", "totalParams": 35000000000, "activeParams": 3000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "20 KiB", "kvCacheKiB": 20, "kvCacheLevel": "Very low", "summary": "Compact Qwen3.6 open-weight MoE that keeps the Qwen3.5 hybrid Gated DeltaNet/Gated Attention recipe while activating only about 3B parameters.", "keyDetail": "Uses 256 experts with 8 routed plus 1 shared expert active inside a 40-layer hybrid stack.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-6-35b-a3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-6-35b-a3b.webp", "alt": "Figure 1: Qwen3.6 35B-A3B", "width": 960, "height": 904 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/main/config.json", "title": "Qwen/Qwen3.6-35B-A3B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3.6-35B-A3B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://qwen.ai/blog?id=qwen3.6-35b-a3b", "title": null } ], "concepts": [ { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 18.2, "profile": { "General": 28.7, "Scientific": 11.3, "Coding": 18.3, "Agents": 12.7 }, "url": "https://artificialanalysis.ai/models/qwen3-6-35b-a3b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-6-35b-a3b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-6-35b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-6-35b-a3b.md" }, { "slug": "glm-5-1", "order": 64, "name": "GLM-5.1 (744B)", "baseTitle": "GLM-5.1", "company": "Z.ai", "country": "China", "flag": "🇨🇳", "date": "2026-04-07", "decoderType": "Sparse MoE", "attention": "MLA with DeepSeek Sparse Attention", "layerMix": "78 MLA", "scale": "744B total, 40B active (5.4% active)", "totalParams": 744000000000, "activeParams": 40000000000, "context": "202,752", "contextTokens": 202752, "license": "MIT License", "kvCacheText": "87.8 KiB", "kvCacheKiB": 87.8, "kvCacheLevel": "Moderate", "summary": "Post-trained GLM refresh that keeps the GLM-5 backbone intact but targets stronger long-horizon agentic coding.", "keyDetail": "Architecture stays aligned with GLM-5, including its MTP-capable backbone; the main shift is post-training.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/glm-5-1.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/glm-5-1.webp", "alt": "Figure 1: GLM-5.1", "width": 960, "height": 934 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/zai-org/GLM-5.1/blob/main/config.json", "title": "zai-org/GLM-5.1" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2602.15763", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 26.1, "profile": { "General": 34.7, "Scientific": 17.3, "Coding": 23.4, "Agents": 25.3 }, "url": "https://artificialanalysis.ai/models/glm-5-1" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-glm-5-1", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/glm-5-1/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/glm-5-1.md" }, { "slug": "gemma-4-31b", "order": 35, "name": "Gemma 4 (31B)", "baseTitle": "Gemma 4 31B", "company": "Google", "country": "United States", "flag": "🇺🇸", "date": "2026-04-02", "decoderType": "Dense", "attention": "GQA with QK-Norm, unified K/V on global layers, p-RoPE on global layers, and 5:1 sliding-window/global attention", "layerMix": "50 sliding-window + 10 global", "scale": "30.7B parameters", "totalParams": 30700000000, "activeParams": 30700000000, "context": "256,000", "contextTokens": 256000, "license": "Apache License 2.0", "kvCacheText": "840 KiB", "kvCacheKiB": 840, "kvCacheLevel": "Very high", "summary": "Dense Gemma 4 scales the family to a 256K-context multimodal checkpoint without changing the core local-global recipe much.", "keyDetail": "Carries Gemma's unusual pre/post-norm stack into a larger 31B dense model with 256K context.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gemma-4-31b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gemma-4-31b.webp", "alt": "Figure 58: Gemma 4 31B", "width": 918, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A723-gemma-4", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A723-gemma-4", "title": null }, { "label": "config.json", "href": "https://huggingface.co/google/gemma-4-31B-it/blob/main/config.json", "title": "google/gemma-4-31B-it" }, { "label": "Tech report", "href": "https://ai.google.dev/gemma/docs/core/model_card_4", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 13.9, "profile": { "Scientific": 5.9 }, "url": "https://artificialanalysis.ai/models/gemma-4-31b-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gemma-4-31b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/gemma-4-31b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gemma-4-31b.md" }, { "slug": "gemma-4-26b-a4b", "order": 36, "name": "Gemma 4 (26B-A4B)", "baseTitle": "Gemma 4 26B-A4B", "company": "Google", "country": "United States", "flag": "🇺🇸", "date": "2026-04-02", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm, unified K/V on global layers, p-RoPE on global layers, and 5:1 sliding-window/global attention", "layerMix": "25 sliding-window + 5 global", "scale": "25.2B total, 3.8B active (15.1% active)", "totalParams": 25200000000, "activeParams": 3800000000, "context": "256,000", "contextTokens": 256000, "license": "Apache License 2.0", "kvCacheText": "210 KiB", "kvCacheKiB": 210, "kvCacheLevel": "High", "summary": "Sparse Gemma 4 variant that keeps the local:global attention backbone while swapping dense FFNs for MoE layers.", "keyDetail": "Uses 128 total experts with only 8 routed plus 1 shared expert active per token.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gemma-4-26b-a4b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gemma-4-26b-a4b.webp", "alt": "Figure 60: Gemma 4 26B-A4B", "width": 960, "height": 952 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A723-gemma-4", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A723-gemma-4", "title": null }, { "label": "config.json", "href": "https://huggingface.co/google/gemma-4-26B-A4B-it/blob/main/config.json", "title": "google/gemma-4-26B-A4B-it" }, { "label": "Tech report", "href": "https://ai.google.dev/gemma/docs/core/model_card_4", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 13.1, "profile": { "Scientific": 5.7 }, "url": "https://artificialanalysis.ai/models/gemma-4-26b-a4b-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gemma-4-26b-a4b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/gemma-4-26b-a4b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gemma-4-26b-a4b.md" }, { "slug": "gemma-4-e2b", "order": 58, "name": "Gemma 4 (E2B)", "baseTitle": "Gemma 4 E2B", "company": "Google", "country": "United States", "flag": "🇺🇸", "date": "2026-04-02", "decoderType": "Dense", "attention": "Multi-query attention with QK-Norm, unified K/V on global layers, p-RoPE on global layers, and 4:1 sliding-window/global attention", "layerMix": "28 sliding-window + 7 global", "scale": "5.1B parameters (2.3B effective)", "totalParams": 5100000000, "activeParams": 5100000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "35 KiB", "kvCacheKiB": 35, "kvCacheLevel": "Low", "summary": "Smallest Gemma 4 edge model keeps the family's hybrid attention stack and adds native audio on a phone-scale multimodal footprint. Uses per-layer embeddings, which add small layer-specific token vectors without scaling the full compute path, so its compute footprint is closer to 2.3B than a full 5.1B dense model.", "keyDetail": "Uses a double-wide GELU MLP plus a single KV head to stay light enough for offline edge deployments.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gemma-4-e2b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gemma-4-e2b.webp", "alt": "Figure 2: Gemma 4 E2B", "width": 918, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A71-reusing-kv-tensors-across-layers-to-shrink-the-cache-gemma-4", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A71-reusing-kv-tensors-across-layers-to-shrink-the-cache-gemma-4", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/blob/main/ch05/17_gemma4/standalone-gemma4-plus-kvcache.ipynb", "title": null }, { "label": "config.json", "href": "https://huggingface.co/google/gemma-4-E2B-it/blob/main/config.json", "title": "google/gemma-4-E2B-it" }, { "label": "Tech report", "href": "https://ai.google.dev/gemma/docs/core/model_card_4", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "kv-sharing", "label": "KV Sharing", "name": "Cross-Layer KV Sharing", "url": "https://llm-atlas-7or.pages.dev/concepts/kv-sharing/" }, { "slug": "per-layer-embeddings", "label": "PLE", "name": "Per-Layer Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/per-layer-embeddings/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 6.5, "profile": { "Scientific": 2.4 }, "url": "https://artificialanalysis.ai/models/gemma-4-e2b-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gemma-4-e2b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/gemma-4-e2b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gemma-4-e2b.md" }, { "slug": "gemma-4-e4b", "order": 59, "name": "Gemma 4 (E4B)", "baseTitle": "Gemma 4 E4B", "company": "Google", "country": "United States", "flag": "🇺🇸", "date": "2026-04-02", "decoderType": "Dense", "attention": "GQA with QK-Norm, unified K/V on global layers, p-RoPE on global layers, and 5:1 sliding-window/global attention", "layerMix": "35 sliding-window + 7 global", "scale": "8B parameters (4.5B effective)", "totalParams": 8000000000, "activeParams": 8000000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "84 KiB", "kvCacheKiB": 84, "kvCacheLevel": "Moderate", "summary": "Larger Gemma 4 edge variant keeps the same multimodal hybrid recipe but doubles width and KV heads for a stronger 128K mobile checkpoint. Uses per-layer embeddings, which add small layer-specific token vectors without scaling the full compute path, so its compute footprint is closer to 4.5B than a full 8B dense model.", "keyDetail": "Steps up to a 42-layer stack with 2 KV heads while keeping the same edge-oriented local/global template.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gemma-4-e4b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gemma-4-e4b.webp", "alt": "Figure 2: Gemma 4 E4B", "width": 918, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A71-reusing-kv-tensors-across-layers-to-shrink-the-cache-gemma-4", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/recent-developments-in-llm-architectures#%C2%A71-reusing-kv-tensors-across-layers-to-shrink-the-cache-gemma-4", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/blob/main/ch05/17_gemma4/standalone-gemma4-plus-kvcache.ipynb", "title": null }, { "label": "config.json", "href": "https://huggingface.co/google/gemma-4-E4B-it/blob/main/config.json", "title": "google/gemma-4-E4B-it" }, { "label": "Tech report", "href": "https://ai.google.dev/gemma/docs/core/model_card_4", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "kv-sharing", "label": "KV Sharing", "name": "Cross-Layer KV Sharing", "url": "https://llm-atlas-7or.pages.dev/concepts/kv-sharing/" }, { "slug": "per-layer-embeddings", "label": "PLE", "name": "Per-Layer Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/per-layer-embeddings/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 7.5, "profile": { "Scientific": 2.6 }, "url": "https://artificialanalysis.ai/models/gemma-4-e4b-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gemma-4-e4b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/gemma-4-e4b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gemma-4-e4b.md" }, { "slug": "minimax-m2-7-230b", "order": 57, "name": "MiniMax M2.7 (230B)", "baseTitle": "MiniMax M2.7 230B", "company": "MiniMax", "country": "China", "flag": "🇨🇳", "date": "2026-03-18", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm", "layerMix": "62 GQA", "scale": "230B total, 10B active (4.3% active)", "totalParams": 230000000000, "activeParams": 10000000000, "context": "196,608", "contextTokens": 196608, "license": "MiniMax M2.7 Non-Commercial License", "kvCacheText": "248 KiB", "kvCacheKiB": 248, "kvCacheLevel": "High", "summary": "Agent-focused 230B follow-up built for self-evolution, agent harnesses, and productivity workflows.", "keyDetail": "Keeps the M2.5-style 62-layer sparse MoE stack and adds three MTP modules for speculative decoding support.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/minimax-m2-7-230b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/minimax-m2-7-230b.webp", "alt": "Figure 1: MiniMax M2.7 230B", "width": 960, "height": 957 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/MiniMaxAI/MiniMax-M2.7/blob/main/config.json", "title": "MiniMaxAI/MiniMax-M2.7" }, { "label": "License", "href": "https://huggingface.co/MiniMaxAI/MiniMax-M2.7/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2605.26494", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 22.8, "profile": { "General": 34.7, "Scientific": 15.1, "Coding": 25.1, "Agents": 14.7 }, "url": "https://artificialanalysis.ai/models/minimax-m2-7" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-minimax-m2-7-230b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/minimax-m2-7-230b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/minimax-m2-7-230b.md" }, { "slug": "mistral-small-4", "order": 55, "name": "Mistral Small 4 (119B)", "baseTitle": "Mistral Small 4", "company": "Mistral AI", "country": "France", "flag": "🇫🇷", "date": "2026-03-16", "decoderType": "Sparse MoE", "attention": "MLA", "layerMix": "36 MLA", "scale": "119B total, 6.63B active (5.6% active)", "totalParams": 119000000000, "activeParams": 6630000000, "context": "256,000", "contextTokens": 256000, "license": "Apache License 2.0", "kvCacheText": "22.5 KiB", "kvCacheKiB": 22.5, "kvCacheLevel": "Very low", "summary": "Multimodal Mistral Small refresh that jumps from the older dense 24B stack to an MLA-based sparse MoE design.", "keyDetail": "Uses 128 experts with 4 routed plus 1 shared expert active per token while unifying instruct, reasoning, and vision.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/mistral-small-4.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/mistral-small-4.webp", "alt": "Figure 1: Mistral Small 4", "width": 960, "height": 938 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603/blob/main/config.json", "title": "mistralai/Mistral-Small-4-119B-2603" }, { "label": "Tech report", "href": "https://mistral.ai/news/mistral-small-4", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 4.3, "profile": { "Knowledge Work": 0.2, "Coding": 0, "Science": 0, "General": 17 }, "url": "https://artificialanalysis.ai/models/mistral-small-4" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 11.3, "profile": { "General": 21.1, "Scientific": 5.1, "Coding": 19.4, "Agents": 0.2 }, "url": "https://artificialanalysis.ai/models/mistral-small-4" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-mistral-small-4", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/mistral-small-4/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/mistral-small-4.md" }, { "slug": "nemotron-3-nano-4b", "order": 56, "name": "Nemotron 3 Nano (4B)", "baseTitle": "Nemotron 3 Nano 4B", "company": "NVIDIA", "country": "United States", "flag": "🇺🇸", "date": "2026-03-16", "decoderType": "Dense hybrid", "attention": "GQA with only 4 attention layers", "layerMix": "4 GQA + 21 Mamba-2 + 17 FFN", "scale": "4B parameters", "totalParams": 4000000000, "activeParams": 4000000000, "context": "262,144", "contextTokens": 262144, "license": "NVIDIA Nemotron Open Model License", "kvCacheText": "16 KiB", "kvCacheKiB": 16, "kvCacheLevel": "Very low", "summary": "Compact on-device hybrid that compresses Nemotron Nano 9B v2 into a mostly Mamba-2 stack with only four attention layers.", "keyDetail": "Uses a 42-layer stack with 21 Mamba-2 blocks, 17 ReLU² FFNs, and just 4 GQA layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/nemotron-3-nano-4b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/nemotron-3-nano-4b.webp", "alt": "Figure 1: Nemotron 3 Nano 4B", "width": 960, "height": 855 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/blob/main/config.json", "title": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16" }, { "label": "License", "href": "https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-nemotron-open-model-license/", "title": null }, { "label": "Tech report", "href": "https://huggingface.co/blog/nvidia/nemotron-3-nano-4b", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" } ], "aai": { "v5": { "label": "AA Intelligence Index v5.0", "total": 1.8, "profile": { "Knowledge Work": 0.2, "Coding": 0, "Science": 0, "General": 7.1 }, "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-nano-4b" }, "v4": { "label": "AA Intelligence Index v4.3.2", "total": 4.8, "profile": { "General": 8.8, "Scientific": 2.4, "Coding": 8, "Agents": 0.1 }, "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-nano-4b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-nemotron-3-nano-4b", "family": "hybrid", "url": "https://llm-atlas-7or.pages.dev/models/nemotron-3-nano-4b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/nemotron-3-nano-4b.md" }, { "slug": "nemotron-3-super-120b-a12b", "order": 34, "name": "Nemotron 3 Super (120B-A12B)", "baseTitle": "Nemotron 3 Super 120B-A12B", "company": "NVIDIA", "country": "United States", "flag": "🇺🇸", "date": "2026-03-11", "decoderType": "Hybrid MoE", "attention": "Mostly Mamba-2 with a few GQA layers", "layerMix": "8 GQA + 40 Mamba-2 + 40 MoE", "scale": "120B total, 12B active (10% active)", "totalParams": 120000000000, "activeParams": 12000000000, "context": "1,000,000", "contextTokens": 1000000, "license": "NVIDIA Nemotron Open Model License", "kvCacheText": "8 KiB", "kvCacheKiB": 8, "kvCacheLevel": "Very low", "summary": "The Super variant scales up Nano and adds both latent experts and native speculative decoding support.", "keyDetail": "Adds latent-space MoE and shared-weight MTP for fast inference.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/nemotron-3-super-120b-a12b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/nemotron-3-super-120b-a12b.webp", "alt": "Figure 51.3: Nemotron 3 Super 120B-A12B", "width": 941, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7182-nemotron-3-super", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7182-nemotron-3-super", "title": null }, { "label": "config.json", "href": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/blob/main/config.json", "title": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" }, { "label": "License", "href": "https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-nemotron-open-model-license/", "title": null }, { "label": "Tech report", "href": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Super-Technical-Report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "latent-moe", "label": "LatentMoE", "name": "Latent MoE", "url": "https://llm-atlas-7or.pages.dev/concepts/latent-moe/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 12.8, "profile": { "General": 22.1, "Scientific": 12, "Coding": 18.1, "Agents": 0.6 }, "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-super-120b-a12b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-nemotron-3-super-120b-a12b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/nemotron-3-super-120b-a12b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/nemotron-3-super-120b-a12b.md" }, { "slug": "sarvam-30b", "order": 47, "name": "Sarvam (30B)", "baseTitle": "Sarvam 30B", "company": "Sarvam AI", "country": "India", "flag": "🇮🇳", "date": "2026-03-03", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm", "layerMix": "19 GQA", "scale": "30B total, 2.4B active (8% active)", "totalParams": 30000000000, "activeParams": 2400000000, "context": "131,072", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "19 KiB", "kvCacheKiB": 19, "kvCacheLevel": "Very low", "summary": "Reasoning-oriented Indian-language sparse MoE that keeps GQA at the smaller size.", "keyDetail": "Large vocabulary and strong Indic language support paired with a reasoning-focused sparse MoE design.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/sarvam-30b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/sarvam-30b.webp", "alt": "Figure 38: Sarvam 30B", "width": 960, "height": 930 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A7update-1-sarvam-30b-and-105b-mar-6-2026", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A7update-1-sarvam-30b-and-105b-mar-6-2026", "title": null }, { "label": "config.json", "href": "https://huggingface.co/sarvamai/sarvam-30b/blob/main/config.json", "title": "sarvamai/sarvam-30b" }, { "label": "Tech report", "href": "https://www.sarvam.ai/blogs/sarvam-30b-105b", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 6.6, "profile": { "Scientific": 3.9 }, "url": "https://artificialanalysis.ai/models/sarvam-30b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-sarvam-30b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/sarvam-30b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/sarvam-30b.md" }, { "slug": "sarvam-105b", "order": 48, "name": "Sarvam (105B)", "baseTitle": "Sarvam 105B", "company": "Sarvam AI", "country": "India", "flag": "🇮🇳", "date": "2026-03-03", "decoderType": "Sparse MoE", "attention": "MLA with KV LayerNorm and NoPE + RoPE", "layerMix": "32 MLA", "scale": "105B total, 10.3B active (9.8% active)", "totalParams": 105000000000, "activeParams": 10300000000, "context": "131,072", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "36 KiB", "kvCacheKiB": 36, "kvCacheLevel": "Low", "summary": "Larger Sarvam variant keeps the sparse MoE layout but switches from GQA to MLA.", "keyDetail": "Large vocabulary and strong Indic language support carried into the larger MLA-based sparse MoE variant.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/sarvam-105b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/sarvam-105b.webp", "alt": "Figure 38: Sarvam 105B", "width": 960, "height": 909 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A7update-1-sarvam-30b-and-105b-mar-6-2026", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A7update-1-sarvam-30b-and-105b-mar-6-2026", "title": null }, { "label": "config.json", "href": "https://huggingface.co/sarvamai/sarvam-105b/blob/main/config.json", "title": "sarvamai/sarvam-105b" }, { "label": "Tech report", "href": "https://www.sarvam.ai/blogs/sarvam-30b-105b", "title": null } ], "concepts": [ { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 8.8, "profile": { "Scientific": 5.5 }, "url": "https://artificialanalysis.ai/models/sarvam-105b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-sarvam-105b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/sarvam-105b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/sarvam-105b.md" }, { "slug": "qwen3-5-397b", "order": 46, "name": "Qwen3.5 (397B)", "baseTitle": "Qwen3.5 397B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2026-02-16", "decoderType": "Sparse hybrid", "attention": "3:1 Gated DeltaNet and Gated Attention", "layerMix": "15 gated attention + 45 DeltaNet", "scale": "397B total, 17B active (4.3% active)", "totalParams": 397000000000, "activeParams": 17000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "30 KiB", "kvCacheKiB": 30, "kvCacheLevel": "Low", "summary": "Mainline Qwen refresh that brings the Next-style hybrid attention into the flagship series.", "keyDetail": "Turns the former Qwen3-Next side branch into the new core design with 512 experts and 17B active parameters.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-5-397b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-5-397b.webp", "alt": "Figure 31: Qwen3.5 397B", "width": 960, "height": 904 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A78-qwen35-and-the-continuation-of-hybrid-attention", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A78-qwen35-and-the-continuation-of-hybrid-attention", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/16_qwen3.5", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/main/config.json", "title": "Qwen/Qwen3.5-397B-A17B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 21.4, "profile": { "Scientific": 10.3 }, "url": "https://artificialanalysis.ai/models/qwen3-5-397b-a17b-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-5-397b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-5-397b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-5-397b.md" }, { "slug": "ling-2-5-1t", "order": 45, "name": "Ling 2.5 (1T)", "baseTitle": "Ling 2.5 1T", "company": "InclusionAI", "country": "China", "flag": "🇨🇳", "date": "2026-02-15", "decoderType": "Sparse hybrid", "attention": "Lightning Attention plus MLA", "layerMix": "10 MLA + 70 Lightning Attention", "scale": "1T total, 63B active (6.3% active)", "totalParams": 1000000000000, "activeParams": 63000000000, "context": "256,000", "contextTokens": 256000, "license": "MIT License", "kvCacheText": "11.2 KiB", "kvCacheKiB": 11.2, "kvCacheLevel": "Very low", "summary": "Trillion-parameter long-context model that swaps DeltaNet for Lightning Attention.", "keyDetail": "Uses a 7:1 linear-attention/MLA ratio and a much larger 63B active path.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/ling-2-5-1t.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/ling-2-5-1t.webp", "alt": "Figure 32: Ling 2.5 1T", "width": 960, "height": 923 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A79-ant-groups-ling-25-1t-with-lightning-attention", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A79-ant-groups-ling-25-1t-with-lightning-attention", "title": null }, { "label": "config.json", "href": "https://huggingface.co/inclusionAI/Ling-2.5-1T/blob/main/config.json", "title": "inclusionAI/Ling-2.5-1T" } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-ling-2-5-1t", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/ling-2-5-1t/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/ling-2-5-1t.md" }, { "slug": "tiny-aya-3-35b", "order": 44, "name": "Tiny Aya (3.35B)", "baseTitle": "Tiny Aya 3.35B", "company": "Cohere", "country": "Canada", "flag": "🇨🇦", "date": "2026-02-13", "decoderType": "Dense", "attention": "GQA with 3:1 sliding-window attention", "layerMix": "27 sliding-window + 9 global", "scale": "3.35B parameters", "totalParams": 3350000000, "activeParams": 3350000000, "context": "8,192", "contextTokens": 8192, "license": "Creative Commons Attribution-NonCommercial 4.0", "kvCacheText": "72 KiB", "kvCacheKiB": 72, "kvCacheLevel": "Low", "summary": "Compact multilingual model from Cohere with a rare parallel transformer block.", "keyDetail": "Runs attention and the MLP in parallel while mixing RoPE with NoPE.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/tiny-aya-3-35b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/tiny-aya-3-35b.webp", "alt": "Figure 35: Tiny Aya 3.35B", "width": 960, "height": 938 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A710-tiny-aya-a-335b-model-with-strong-multilingual-support", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A710-tiny-aya-a-335b-model-with-strong-multilingual-support", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/15_tiny-aya", "title": null }, { "label": "config.json", "href": "https://huggingface.co/CohereLabs/tiny-aya-base/blob/main/config.json", "title": "CohereLabs/tiny-aya-base" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2603.11510", "title": null } ], "concepts": [ { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 4.8, "profile": { "Scientific": 2.6 }, "url": "https://artificialanalysis.ai/models/tiny-aya-global" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-tiny-aya-3-35b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/tiny-aya-3-35b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/tiny-aya-3-35b.md" }, { "slug": "minimax-m2-5-230b", "order": 43, "name": "MiniMax-M2.5 (230B)", "baseTitle": "MiniMax M2.5 230B", "company": "MiniMax", "country": "China", "flag": "🇨🇳", "date": "2026-02-12", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm", "layerMix": "62 GQA", "scale": "230B total, 10B active (4.3% active)", "totalParams": 230000000000, "activeParams": 10000000000, "context": "196,608", "contextTokens": 196608, "license": "Modified MIT License", "kvCacheText": "248 KiB", "kvCacheKiB": 248, "kvCacheLevel": "High", "summary": "Popular 230B coder that opts for a classic architecture instead of the newer hybrid-attention ideas.", "keyDetail": "Deliberately avoids sliding-window or linear-attention hybrids while keeping a 10B active path and three MTP modules.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/minimax-m2-5-230b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/minimax-m2-5-230b.webp", "alt": "Figure 26: MiniMax M2.5 230B", "width": 960, "height": 957 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A76-minimax-m25-a-strong-coder-with-only-230b-parameters", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A76-minimax-m25-a-strong-coder-with-only-230b-parameters", "title": null }, { "label": "config.json", "href": "https://huggingface.co/MiniMaxAI/MiniMax-M2.5/blob/main/config.json", "title": "MiniMaxAI/MiniMax-M2.5" }, { "label": "Tech report", "href": "https://arxiv.org/abs/2605.26494", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 22.8, "profile": { "Scientific": 10.8 }, "url": "https://artificialanalysis.ai/models/minimax-m2-5" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-minimax-m2-5-230b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/minimax-m2-5-230b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/minimax-m2-5-230b.md" }, { "slug": "glm-5-744b", "order": 33, "name": "GLM-5 (744B)", "baseTitle": "GLM-5 744B", "company": "Z.ai", "country": "China", "flag": "🇨🇳", "date": "2026-02-11", "decoderType": "Sparse MoE", "attention": "MLA with DeepSeek Sparse Attention", "layerMix": "78 MLA", "scale": "744B total, 40B active (5.4% active)", "totalParams": 744000000000, "activeParams": 40000000000, "context": "202,752", "contextTokens": 202752, "license": "MIT License", "kvCacheText": "87.8 KiB", "kvCacheKiB": 87.8, "kvCacheLevel": "Moderate", "summary": "Huge GLM refresh that adopts both MLA and DeepSeek Sparse Attention for flagship-scale inference.", "keyDetail": "Bigger than GLM-4.7, with more experts, fewer layers, and DeepSeek-style MTP.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/glm-5-744b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/glm-5-744b.webp", "alt": "Figure 56: GLM-5 744B", "width": 960, "height": 934 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A721-glm-5", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A721-glm-5", "title": null }, { "label": "config.json", "href": "https://huggingface.co/zai-org/GLM-5/blob/main/config.json", "title": "zai-org/GLM-5" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2602.15763", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 21.8, "profile": { "Scientific": 3.8 }, "url": "https://artificialanalysis.ai/models/glm-5-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-glm-5-744b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/glm-5-744b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/glm-5-744b.md" }, { "slug": "nanbeige-4-1-3b", "order": 42, "name": "Nanbeige 4.1 (3B)", "baseTitle": "Nanbeige 4.1 3B", "company": "BOSS Zhipin", "country": "China", "flag": "🇨🇳", "date": "2026-02-10", "decoderType": "Dense", "attention": "GQA", "layerMix": "32 GQA", "scale": "3B parameters", "totalParams": 3000000000, "activeParams": 3000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "64 KiB", "kvCacheKiB": 64, "kvCacheLevel": "Low", "summary": "Small on-device oriented model that stays close to Llama 3.2 while nudging the scaling choices.", "keyDetail": "Llama-like stack without tying input embeddings to the output layer.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/nanbeige-4-1-3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/nanbeige-4-1-3b.webp", "alt": "Figure 29: Nanbeige 4.1 3B", "width": 951, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A77-nanbeige-41-3b-a-strong-llama-3-successor", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A77-nanbeige-41-3b-a-strong-llama-3-successor", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Nanbeige/Nanbeige4.1-3B/blob/main/config.json", "title": "Nanbeige/Nanbeige4.1-3B" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2602.13367", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 8.4, "profile": { "Scientific": 5.4 }, "url": "https://artificialanalysis.ai/models/nanbeige4-1-3b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-nanbeige-4-1-3b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/nanbeige-4-1-3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/nanbeige-4-1-3b.md" }, { "slug": "step-3-5-flash-196b", "order": 41, "name": "Step 3.5 Flash (196B)", "baseTitle": "Step 3.5 Flash 196B", "company": "StepFun", "country": "China", "flag": "🇨🇳", "date": "2026-02-01", "decoderType": "Sparse MoE", "attention": "GQA with 3:1 sliding-window attention", "layerMix": "36 sliding-window + 12 global", "scale": "196B total, 11B active (5.6% active)", "totalParams": 196000000000, "activeParams": 11000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "192 KiB", "kvCacheKiB": 192, "kvCacheLevel": "High", "summary": "Throughput-oriented MoE model that stays competitive with much larger DeepSeek-style systems.", "keyDetail": "Uses MTP-3 during both training and inference for unusually high throughput.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/step-3-5-flash-196b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/step-3-5-flash-196b.webp", "alt": "Figure 12: Step 3.5 Flash 196B", "width": 960, "height": 940 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A73-stepfuns-step-35-flash-good-performance-at-great-tokens-sec-throughput", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A73-stepfuns-step-35-flash-good-performance-at-great-tokens-sec-throughput", "title": null }, { "label": "config.json", "href": "https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/main/config.json", "title": "stepfun-ai/Step-3.5-Flash" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2602.10604", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 17, "profile": { "Scientific": 13.4 }, "url": "https://artificialanalysis.ai/models/step-3-5-flash" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-step-3-5-flash-196b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/step-3-5-flash-196b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/step-3-5-flash-196b.md" }, { "slug": "longcat-flash-lite-68-5b-a3b", "order": 54, "name": "LongCat-Flash-Lite (68.5B-A3B)", "baseTitle": "LongCat-Flash-Lite 68.5B-A3B", "company": "Meituan", "country": "China", "flag": "🇨🇳", "date": "2026-01-28", "decoderType": "Sparse MoE", "attention": "MLA with RoPE + NoPE", "layerMix": "14 logical layers / 28 MLA sublayers", "scale": "68.5B total, ~3B active avg (4.4% active)", "totalParams": 68500000000, "activeParams": 3000000000, "context": "256,000", "contextTokens": 256000, "license": "MIT License", "kvCacheText": "31.5 KiB", "kvCacheKiB": 31.5, "kvCacheLevel": "Low", "summary": "Meituan's LongCat-Next text backbone is a lightweight MoE that shifts a large share of parameters into N-gram embeddings while keeping sparse inference around 3B active parameters.", "keyDetail": "Uses N-gram embedding expansion, Shortcut MoE routing over 256 routed plus 128 zero/identity experts, and top-k 12 expert choices per token.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/longcat-flash-lite-68-5b-a3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/longcat-flash-lite-68-5b-a3b.webp", "alt": "Figure 1: LongCat-Flash-Lite 68.5B-A3B", "width": 879, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/meituan-longcat/LongCat-Flash-Lite/blob/main/config.json", "title": "meituan-longcat/LongCat-Flash-Lite" }, { "label": "Tech report", "href": "https://arxiv.org/abs/2601.21204", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 11.5, "profile": { "Scientific": 2.9 }, "url": "https://artificialanalysis.ai/models/longcat-flash-lite" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-longcat-flash-lite-68-5b-a3b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/longcat-flash-lite-68-5b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/longcat-flash-lite-68-5b-a3b.md" }, { "slug": "arcee-ai-trinity-large-400b", "order": 32, "name": "Arcee AI Trinity Large (400B)", "baseTitle": "Arcee AI Trinity Large 400B", "company": "Arcee AI", "country": "United States", "flag": "🇺🇸", "date": "2026-01-27", "decoderType": "Sparse MoE", "attention": "GQA with gated attention and 3:1 sliding-window/global attention", "layerMix": "45 sliding-window + 15 global", "scale": "400B total, 13B active (3.3% active)", "totalParams": 400000000000, "activeParams": 13000000000, "context": "512,000", "contextTokens": 512000, "license": "Apache License 2.0", "kvCacheText": "240 KiB", "kvCacheKiB": 240, "kvCacheLevel": "High", "summary": "Arcee's flagship blends several efficiency tricks into a DeepSeek-like coarse MoE design; the AA score shown here refers to the Trinity Large Thinking checkpoint.", "keyDetail": "Combines QK-Norm, RoPE+NoPE, sandwich norm, and a coarse-grained MoE.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/arcee-ai-trinity-large-400b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/arcee-ai-trinity-large-400b.webp", "alt": "Figure 53: Arcee AI Trinity Large 400B", "width": 944, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A720-arcee-ai-trinity-large", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A720-arcee-ai-trinity-large", "title": null }, { "label": "config.json", "href": "https://huggingface.co/arcee-ai/Trinity-Large-Base/blob/main/config.json", "title": "arcee-ai/Trinity-Large-Base" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2602.17004", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 10.8, "profile": { "General": 16.6, "Scientific": 8.4, "Coding": 20.6, "Agents": 0.2 }, "url": "https://artificialanalysis.ai/models/trinity-large-thinking" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-arcee-ai-trinity-large-400b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/arcee-ai-trinity-large-400b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/arcee-ai-trinity-large-400b.md" }, { "slug": "kimi-k2-5", "order": 40, "name": "Kimi K2.5 (1T)", "baseTitle": "Kimi K2.5", "company": "Moonshot AI", "country": "China", "flag": "🇨🇳", "date": "2026-01-27", "decoderType": "Sparse MoE", "attention": "MLA", "layerMix": "61 MLA", "scale": "1T total, 32B active (3.2% active)", "totalParams": 1000000000000, "activeParams": 32000000000, "context": "256,000", "contextTokens": 256000, "license": "Modified MIT License", "kvCacheText": "68.6 KiB", "kvCacheKiB": 68.6, "kvCacheLevel": "Low", "summary": "Native-multimodal Moonshot flagship that keeps the K2/DeepSeek-style MoE layout and pushes native context to 256k.", "keyDetail": "Keeps the 384-expert K2 backbone, but adds multimodal capabilities (not shown) and doubles the native context length.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/kimi-k2-5.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/kimi-k2-5.webp", "alt": "Figure 8: Kimi K2.5", "width": 960, "height": 930 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A72-moonshot-ais-kimi-k25-a-deepseek-like-model-at-a-1-trillion-parameter-scale", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A72-moonshot-ais-kimi-k25-a-deepseek-like-model-at-a-1-trillion-parameter-scale", "title": null }, { "label": "config.json", "href": "https://huggingface.co/moonshotai/Kimi-K2.5/blob/main/config.json", "title": "moonshotai/Kimi-K2.5" }, { "label": "License", "href": "https://huggingface.co/moonshotai/Kimi-K2.5/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2602.02276", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 19.4, "profile": { "Scientific": 6.9 }, "url": "https://artificialanalysis.ai/models/kimi-k2-5-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-kimi-k2-5", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/kimi-k2-5/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/kimi-k2-5.md" }, { "slug": "glm-4-7-355b", "order": 31, "name": "GLM-4.7 (355B)", "baseTitle": "GLM-4.7 355B", "company": "Z.ai", "country": "China", "flag": "🇨🇳", "date": "2025-12-22", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm", "layerMix": "92 GQA", "scale": "355B total, 32B active (9% active)", "totalParams": 355000000000, "activeParams": 32000000000, "context": "202,752", "contextTokens": 202752, "license": "MIT License", "kvCacheText": "368 KiB", "kvCacheKiB": 368, "kvCacheLevel": "Very high", "summary": "Immediate GLM predecessor that stays closer to the older GLM-4.5 style before the MLA shift.", "keyDetail": "Serves as the pre-MLA, pre-sparse-attention baseline and uses an MTP-3-style setup.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/glm-4-7-355b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/glm-4-7-355b.webp", "alt": "Figure 57: GLM-4.7 355B", "width": 960, "height": 931 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A721-glm-5", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A721-glm-5", "title": null }, { "label": "config.json", "href": "https://huggingface.co/zai-org/GLM-4.7/blob/main/config.json", "title": "zai-org/GLM-4.7" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2508.06471", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 17.4, "profile": { "Scientific": 3.2 }, "url": "https://artificialanalysis.ai/models/glm-4-7-non-reasoning" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-glm-4-7-355b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/glm-4-7-355b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/glm-4-7-355b.md" }, { "slug": "xiaomi-mimo-v2-flash-309b", "order": 30, "name": "Xiaomi MiMo-V2-Flash (309B)", "baseTitle": "Xiaomi MiMo-V2-Flash 309B", "company": "Xiaomi", "country": "China", "flag": "🇨🇳", "date": "2025-12-16", "decoderType": "Sparse MoE", "attention": "5:1 sliding-window/global attention", "layerMix": "40 sliding-window + 8 global", "scale": "309B total, 15B active (4.9% active)", "totalParams": 309000000000, "activeParams": 15000000000, "context": "262,144", "contextTokens": 262144, "license": "MIT License", "kvCacheText": "144 KiB", "kvCacheKiB": 144, "kvCacheLevel": "Moderate", "summary": "Large MoE model that pushes sliding-window attention harder than most contemporaries.", "keyDetail": "Uses an unusually small 128-token local window plus multi-token prediction.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/xiaomi-mimo-v2-flash-309b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/xiaomi-mimo-v2-flash-309b.webp", "alt": "Figure 52: Xiaomi MiMo-V2-Flash 309B", "width": 960, "height": 938 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A719-xiaomi-mimo-v2-flash", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A719-xiaomi-mimo-v2-flash", "title": null }, { "label": "config.json", "href": "https://huggingface.co/XiaomiMiMo/MiMo-V2-Flash/blob/main/config.json", "title": "XiaomiMiMo/MiMo-V2-Flash" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2601.02780", "title": null } ], "concepts": [ { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 16, "profile": { "Scientific": 4.3 }, "url": "https://artificialanalysis.ai/models/mimo-v2-flash" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-xiaomi-mimo-v2-flash-309b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-flash-309b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/xiaomi-mimo-v2-flash-309b.md" }, { "slug": "nemotron-3-nano-30b-a3b", "order": 29, "name": "Nemotron 3 Nano (30B-A3B)", "baseTitle": "Nemotron 3 Nano 30B-A3B", "company": "NVIDIA", "country": "United States", "flag": "🇺🇸", "date": "2025-12-04", "decoderType": "Hybrid MoE", "attention": "Mostly Mamba-2 with a few GQA layers", "layerMix": "6 GQA + 23 Mamba-2 + 23 MoE", "scale": "30B total, 3B active (10% active)", "totalParams": 30000000000, "activeParams": 3000000000, "context": "1,000,000", "contextTokens": 1000000, "license": "NVIDIA Nemotron Open Model License", "kvCacheText": "6 KiB", "kvCacheKiB": 6, "kvCacheLevel": "Very low", "summary": "NVIDIA's Nano model is the most extreme transformer-state-space hybrid in the gallery.", "keyDetail": "Interleaves Mamba-2 and MoE blocks, using attention only sparingly.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/nemotron-3-nano-30b-a3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/nemotron-3-nano-30b-a3b.webp", "alt": "Figure 51.1: Nemotron 3 Nano 30B-A3B", "width": 944, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7181-nemotron-3-nano", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7181-nemotron-3-nano", "title": null }, { "label": "config.json", "href": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/blob/main/config.json", "title": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" }, { "label": "License", "href": "https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-nemotron-open-model-license/", "title": null }, { "label": "Tech report", "href": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3-Nano-Technical-Report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 6.8, "profile": { "Scientific": 2.3 }, "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-nano-30b-a3b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-nemotron-3-nano-30b-a3b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/nemotron-3-nano-30b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/nemotron-3-nano-30b-a3b.md" }, { "slug": "mistral-large-3", "order": 28, "name": "Mistral Large 3 (673B)", "baseTitle": "Mistral Large 3", "company": "Mistral AI", "country": "France", "flag": "🇫🇷", "date": "2025-12-02", "decoderType": "Sparse MoE", "attention": "MLA", "layerMix": "61 MLA", "scale": "673B total, 41B active (6.1% active)", "totalParams": 673000000000, "activeParams": 41000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "68.6 KiB", "kvCacheKiB": 68.6, "kvCacheLevel": "Low", "summary": "Mistral's new flagship effectively adopts the DeepSeek architecture and retunes the expert sizes.", "keyDetail": "Near-clone of DeepSeek V3 with larger experts, fewer routed experts, and multimodal support.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/mistral-3-large-673-billion.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/mistral-3-large-673-billion.webp", "alt": "Figure 50: Mistral Large 3", "width": 960, "height": 942 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A717-mistral-3-large", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A717-mistral-3-large", "title": null }, { "label": "params.json", "href": "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512/blob/main/params.json", "title": "mistralai/Mistral-Large-3-675B-Instruct-2512" } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 9.3, "profile": { "General": 17.1, "Scientific": 2.1, "Coding": 18.3, "Agents": 0.2 }, "url": "https://artificialanalysis.ai/models/mistral-large-3" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-mistral-large-3", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/mistral-large-3/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/mistral-large-3.md" }, { "slug": "deepseek-v3-2", "order": 27, "name": "DeepSeek V3.2 (671B)", "baseTitle": "DeepSeek V3.2", "company": "DeepSeek", "country": "China", "flag": "🇨🇳", "date": "2025-12-01", "decoderType": "Sparse MoE", "attention": "MLA with DeepSeek Sparse Attention", "layerMix": "61 MLA", "scale": "671B total, 37B active (5.5% active)", "totalParams": 671000000000, "activeParams": 37000000000, "context": "128,000", "contextTokens": 128000, "license": "MIT License", "kvCacheText": "68.6 KiB", "kvCacheKiB": 68.6, "kvCacheLevel": "Low", "summary": "DeepSeek's successor keeps the V3 template but adds sparse attention to cut long-context costs.", "keyDetail": "Adds sparse attention while retaining the DeepSeek V3-style MTP path.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/deepseek-v3-2-671b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/deepseek-v3-2-671b.webp", "alt": "Figure 48: DeepSeek V3.2", "width": 960, "height": 939 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A716-deepseek-v32", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A716-deepseek-v32", "title": null }, { "label": "config.json", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/config.json", "title": "deepseek-ai/DeepSeek-V3.2" }, { "label": "License", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2512.02556", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "deepseek-sparse-attention", "label": "DSA", "name": "DeepSeek Sparse Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/deepseek-sparse-attention/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 16, "profile": { "Scientific": 6 }, "url": "https://artificialanalysis.ai/models/deepseek-v3-2" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-deepseek-v3-2", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/deepseek-v3-2/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/deepseek-v3-2.md" }, { "slug": "intellect-3", "order": 53, "name": "INTELLECT-3 (106B)", "baseTitle": "INTELLECT-3", "company": "Prime Intellect", "country": "United States", "flag": "🇺🇸", "date": "2025-11-26", "decoderType": "Sparse MoE", "attention": "GQA", "layerMix": "46 GQA", "scale": "106B total, 12B active (11.3% active)", "totalParams": 106000000000, "activeParams": 12000000000, "context": "128,000", "contextTokens": 128000, "license": "MIT License", "kvCacheText": "184 KiB", "kvCacheKiB": 184, "kvCacheLevel": "High", "summary": "Large-scale RL post-training of GLM-4.5-Air that keeps the compact 106B sparse MoE backbone intact.", "keyDetail": "Keeps the GLM-4.5-Air architecture unchanged and shifts the capability profile through SFT plus large-scale RL.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/intellect-3.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/intellect-3.webp", "alt": "Figure 1: INTELLECT-3", "width": 960, "height": 936 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/PrimeIntellect/INTELLECT-3/blob/main/config.json", "title": "PrimeIntellect/INTELLECT-3" }, { "label": "Tech report", "href": "https://storage.googleapis.com/intellect-3-paper/INTELLECT_3_Technical_Report.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 10.6, "profile": { "Scientific": 6.7 }, "url": "https://artificialanalysis.ai/models/intellect-3" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-intellect-3", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/intellect-3/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/intellect-3.md" }, { "slug": "olmo-3-32b", "order": 25, "name": "OLMo 3 (32B)", "baseTitle": "OLMo 3 32B", "company": "Ai2", "country": "United States", "flag": "🇺🇸", "date": "2025-11-20", "decoderType": "Dense", "attention": "GQA with QK-Norm and 3:1 sliding-window/global attention", "layerMix": "48 sliding-window + 16 global", "scale": "32B parameters", "totalParams": 32000000000, "activeParams": 32000000000, "context": "65,536", "contextTokens": 65536, "license": "Apache License 2.0", "kvCacheText": "256 KiB", "kvCacheKiB": 256, "kvCacheLevel": "High", "summary": "Scaled-up OLMo 3 keeps the same block design but moves to grouped-query attention.", "keyDetail": "Keeps post-norm while scaling width and applying YaRN only on global layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/olmo-3-32b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/olmo-3-32b.webp", "alt": "Figure 45: OLMo 3 32B", "width": 924, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A715-olmo-3-thinking", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A715-olmo-3-thinking", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/13_olmo3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/allenai/Olmo-3-32B-Think/blob/main/config.json", "title": "allenai/Olmo-3-32B-Think" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2512.13961", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 6.5, "profile": { "Scientific": 3.2 }, "url": "https://artificialanalysis.ai/models/olmo-3-32b-think" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-olmo-3-32b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/olmo-3-32b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/olmo-3-32b.md" }, { "slug": "olmo-3-7b", "order": 26, "name": "OLMo 3 (7B)", "baseTitle": "OLMo 3 7B", "company": "Ai2", "country": "United States", "flag": "🇺🇸", "date": "2025-11-20", "decoderType": "Dense", "attention": "MHA with QK-Norm and 3:1 sliding-window/global attention", "layerMix": "24 sliding-window + 8 global", "scale": "7B parameters", "totalParams": 7000000000, "activeParams": 7000000000, "context": "65,536", "contextTokens": 65536, "license": "Apache License 2.0", "kvCacheText": "512 KiB", "kvCacheKiB": 512, "kvCacheLevel": "Very high", "summary": "New transparent Allen AI model that keeps OLMo's post-norm flavor while modernizing context handling.", "keyDetail": "Retains post-norm, keeps MHA, and applies YaRN only on global layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/olmo-3-7b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/olmo-3-7b.webp", "alt": "Figure 44: OLMo 3 7B", "width": 924, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A715-olmo-3-thinking", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A715-olmo-3-thinking", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/13_olmo3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/allenai/Olmo-3-1025-7B/blob/main/config.json", "title": "allenai/Olmo-3-1025-7B" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2512.13961", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "mha", "label": "MHA", "name": "Multi-Head Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mha/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 5.2, "profile": { "Scientific": 2.9 }, "url": "https://artificialanalysis.ai/models/olmo-3-7b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-olmo-3-7b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/olmo-3-7b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/olmo-3-7b.md" }, { "slug": "kimi-linear-48b-a3b", "order": 24, "name": "Kimi Linear (48B-A3B)", "baseTitle": "Kimi Linear 48B-A3B", "company": "Moonshot AI", "country": "China", "flag": "🇨🇳", "date": "2025-10-30", "decoderType": "Sparse hybrid", "attention": "3:1 Kimi Delta Attention and MLA", "layerMix": "7 MLA + 20 Kimi Delta Attention", "scale": "48B total, 3B active (6.3% active)", "totalParams": 48000000000, "activeParams": 3000000000, "context": "1,000,000", "contextTokens": 1000000, "license": "MIT License", "kvCacheText": "7.9 KiB", "kvCacheKiB": 7.9, "kvCacheLevel": "Very low", "summary": "Linear-attention hybrid that keeps a transformer backbone but replaces most full-attention layers.", "keyDetail": "Uses NoPE in MLA layers, channel-wise KDA gating, and kernel-4 ShortConv preprocessing for Q, K, and V in KDA layers.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/kimi-linear-48b-a3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/kimi-linear-48b-a3b.webp", "alt": "Figure 42: Kimi Linear 48B-A3B", "width": 960, "height": 883 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7144-kimi-linear-vs-qwen3-next", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7144-kimi-linear-vs-qwen3-next", "title": null }, { "label": "config.json", "href": "https://huggingface.co/moonshotai/Kimi-Linear-48B-A3B-Base/blob/main/config.json", "title": "moonshotai/Kimi-Linear-48B-A3B-Base" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2510.26692", "title": null } ], "concepts": [ { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "shortconv", "label": "ShortConv", "name": "Short Convolution", "url": "https://llm-atlas-7or.pages.dev/concepts/shortconv/" }, { "slug": "attention-residuals", "label": "Attention Residuals", "name": "Attention Residuals", "url": "https://llm-atlas-7or.pages.dev/concepts/attention-residuals/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 7.3, "profile": {}, "url": "https://artificialanalysis.ai/models/kimi-linear-48b-a3b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-kimi-linear-48b-a3b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/kimi-linear-48b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/kimi-linear-48b-a3b.md" }, { "slug": "ouro-thinking-2-6b", "order": 52, "name": "Ouro-Thinking (2.6B)", "baseTitle": "Ouro-Thinking 2.6B", "company": "ByteDance", "country": "China", "flag": "🇨🇳", "date": "2025-10-29", "decoderType": "Dense", "attention": "MHA with RoPE", "layerMix": "48 shared MHA layers × 4 passes", "scale": "2.6B parameters", "totalParams": 2600000000, "activeParams": 2600000000, "context": "65,536", "contextTokens": 65536, "license": "Apache License 2.0", "kvCacheText": "1.5 MiB", "kvCacheKiB": 1536, "kvCacheLevel": "Very high", "summary": "ByteDance's dense LoopLM reuses one 48-layer MHA stack across four recurrent passes and trains an exit gate over the intermediate representations.", "keyDetail": "Reuses the same 48-layer MHA stack across four passes, yielding 192 block applications, with sandwich RMSNorm and a learned exit gate.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/ouro-2-6b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/ouro-2-6b.webp", "alt": "Figure 1: Ouro-Thinking 2.6B", "width": 960, "height": 878 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/ByteDance/Ouro-2.6B-Thinking/blob/main/config.json", "title": "ByteDance/Ouro-2.6B-Thinking" }, { "label": "License", "href": "https://huggingface.co/ByteDance/Ouro-2.6B-Thinking/blob/main/README.md#license", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2510.25741", "title": null } ], "concepts": [ { "slug": "looped-depth-sharing", "label": "Looped Transformer", "name": "Looped Transformer", "url": "https://llm-atlas-7or.pages.dev/concepts/looped-depth-sharing/" }, { "slug": "mha", "label": "MHA", "name": "Multi-Head Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mha/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-ouro-thinking-2-6b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/ouro-thinking-2-6b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/ouro-thinking-2-6b.md" }, { "slug": "minimax-m2-230b", "order": 23, "name": "MiniMax M2 (230B)", "baseTitle": "MiniMax M2 230B", "company": "MiniMax", "country": "China", "flag": "🇨🇳", "date": "2025-10-23", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm and partial RoPE", "layerMix": "62 GQA", "scale": "230B total, 10B active (4.3% active)", "totalParams": 230000000000, "activeParams": 10000000000, "context": "196,608", "contextTokens": 196608, "license": "Modified MIT License", "kvCacheText": "248 KiB", "kvCacheKiB": 248, "kvCacheLevel": "High", "summary": "MiniMax's flagship returns to full attention and looks like a leaner, sparser cousin of Qwen3.", "keyDetail": "Uses per-layer QK-Norm, much sparser MoE routing than Qwen3, and MTP during training.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/minimax-m2-230b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/minimax-m2-230b.webp", "alt": "Figure 39: MiniMax M2 230B", "width": 960, "height": 949 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7131-per-layer-qk-norm", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7131-per-layer-qk-norm", "title": null }, { "label": "config.json", "href": "https://huggingface.co/MiniMaxAI/MiniMax-M2/blob/main/config.json", "title": "MiniMaxAI/MiniMax-M2" }, { "label": "Tech report", "href": "https://arxiv.org/abs/2605.26494", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 18.6, "profile": { "Scientific": 7.3 }, "url": "https://artificialanalysis.ai/models/minimax-m2" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-minimax-m2-230b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/minimax-m2-230b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/minimax-m2-230b.md" }, { "slug": "qwen3-next-80b-a3b", "order": 22, "name": "Qwen3 Next (80B-A3B)", "baseTitle": "Qwen3 Next 80B-A3B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2025-09-09", "decoderType": "Sparse hybrid", "attention": "3:1 Gated DeltaNet and Gated Attention", "layerMix": "12 gated attention + 36 DeltaNet", "scale": "80B total, 3B active (3.8% active)", "totalParams": 80000000000, "activeParams": 3000000000, "context": "262,144", "contextTokens": 262144, "license": "Apache License 2.0", "kvCacheText": "24 KiB", "kvCacheKiB": 24, "kvCacheLevel": "Very low", "summary": "Efficiency-focused Qwen refresh that swaps standard attention for a DeltaNet-attention hybrid.", "keyDetail": "Adds many more experts, a shared expert, native 262k context, and MTP for speculative decoding.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-next-80b-a3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-next-80b-a3b.webp", "alt": "Figure 35: Qwen3 Next 80B-A3B", "width": 960, "height": 907 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7121-expert-size-and-number", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A7121-expert-size-and-number", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/main/config.json", "title": "Qwen/Qwen3-Next-80B-A3B-Instruct" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "gated-attention", "label": "Gated Attention", "name": "Gated Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/" }, { "slug": "hybrid-attention", "label": "Hybrid Attention", "name": "Hybrid Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/hybrid-attention/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 9.6, "profile": { "Scientific": 3.8 }, "url": "https://artificialanalysis.ai/models/qwen3-next-80b-a3b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-next-80b-a3b", "family": "hybrid-moe", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-next-80b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-next-80b-a3b.md" }, { "slug": "grok-2-5-270b", "order": 21, "name": "Grok 2.5 (270B)", "baseTitle": "Grok 2.5 270B", "company": "xAI", "country": "United States", "flag": "🇺🇸", "date": "2025-08-22", "decoderType": "Sparse MoE", "attention": "GQA", "layerMix": "64 GQA", "scale": "270B parameters", "totalParams": 270000000000, "activeParams": 270000000000, "context": "131,072", "contextTokens": 131072, "license": "Grok 2 Community License Agreement", "kvCacheText": "256 KiB", "kvCacheKiB": 256, "kvCacheLevel": "High", "summary": "Rare production-model release that shows an older MoE style with fewer, larger experts.", "keyDetail": "Adds an always-on SwiGLU path that effectively behaves like a shared expert.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/grok-2-5-270b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/grok-2-5-270b.webp", "alt": "Figure 32: Grok 2.5 270B", "width": 960, "height": 878 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A710-grok-25", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A710-grok-25", "title": null }, { "label": "config.json", "href": "https://huggingface.co/xai-org/grok-2/blob/main/config.json", "title": "xai-org/grok-2" } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-grok-2-5-270b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/grok-2-5-270b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/grok-2-5-270b.md" }, { "slug": "gemma-3-270m", "order": 20, "name": "Gemma 3 (270M)", "baseTitle": "Gemma 3 270M", "company": "Google", "country": "United States", "flag": "🇺🇸", "date": "2025-08-14", "decoderType": "Dense", "attention": "Multi-query attention with QK-Norm and 5:1 sliding-window/global attention", "layerMix": "15 sliding-window + 3 global", "scale": "270M parameters", "totalParams": 270000000, "activeParams": 270000000, "context": "128,000", "contextTokens": 128000, "license": "Gemma Terms of Use + Gemma Prohibited Use Policy", "kvCacheText": "18 KiB", "kvCacheKiB": 18, "kvCacheLevel": "Very low", "summary": "Tiny Gemma 3 variant that preserves the family's local-global attention recipe at a toy scale.", "keyDetail": "Keeps the Gemma 3 stack shape while shrinking down to 4 attention heads, a single KV head, and the same 262k vocabulary.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gemma-3-270m.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gemma-3-270m.webp", "alt": "Figure 16: Gemma 3 270M", "width": 960, "height": 955 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A74-mistral-small-31", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A74-mistral-small-31", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/12_gemma3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/google/gemma-3-270m/blob/main/config.json", "title": "google/gemma-3-270m" }, { "label": "License", "href": "https://ai.google.dev/gemma/prohibited_use_policy", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2503.19786", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 5.1, "profile": { "Scientific": 1.8 }, "url": "https://artificialanalysis.ai/models/gemma-3-270m" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gemma-3-270m", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/gemma-3-270m/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gemma-3-270m.md" }, { "slug": "gpt-oss-120b", "order": 18, "name": "GPT-OSS (120B)", "baseTitle": "GPT-OSS 120B", "company": "OpenAI", "country": "United States", "flag": "🇺🇸", "date": "2025-08-04", "decoderType": "Sparse MoE", "attention": "GQA with alternating sliding-window and global layers", "layerMix": "18 sliding-window + 18 global", "scale": "117B total, 5.1B active (4.4% active)", "totalParams": 117000000000, "activeParams": 5100000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "72 KiB", "kvCacheKiB": 72, "kvCacheLevel": "Low", "summary": "Larger gpt-oss variant keeps the same alternating-attention recipe as the 20B model.", "keyDetail": "Shared architectural template scaled up for OpenAI's flagship open-weight release.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gpt-oss-120b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gpt-oss-120b.webp", "alt": "Figure 26: GPT-OSS 120B", "width": 960, "height": 896 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A79-gpt-oss", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A79-gpt-oss", "title": null }, { "label": "config.json", "href": "https://huggingface.co/openai/gpt-oss-120b/blob/main/config.json", "title": "openai/gpt-oss-120b" }, { "label": "License", "href": "https://huggingface.co/openai/gpt-oss-120b/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 11.6, "profile": { "General": 18.8, "Scientific": 10.4, "Coding": 17, "Agents": 1.8 }, "url": "https://artificialanalysis.ai/models/gpt-oss-120b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gpt-oss-120b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/gpt-oss-120b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gpt-oss-120b.md" }, { "slug": "gpt-oss-20b", "order": 19, "name": "GPT-OSS (20B)", "baseTitle": "GPT-OSS 20B", "company": "OpenAI", "country": "United States", "flag": "🇺🇸", "date": "2025-08-04", "decoderType": "Sparse MoE", "attention": "GQA with alternating sliding-window and global layers", "layerMix": "12 sliding-window + 12 global", "scale": "21B total, 3.6B active (17.1% active)", "totalParams": 21000000000, "activeParams": 3600000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "48 KiB", "kvCacheKiB": 48, "kvCacheLevel": "Low", "summary": "OpenAI's smaller open-weight MoE model favors width and alternating local/global attention.", "keyDetail": "Wider and shallower than Qwen3, with attention bias and sink mechanisms.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gpt-oss-20b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gpt-oss-20b.webp", "alt": "Figure 26: GPT-OSS 20B", "width": 960, "height": 906 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A79-gpt-oss", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A79-gpt-oss", "title": null }, { "label": "config.json", "href": "https://huggingface.co/openai/gpt-oss-20b/blob/main/config.json", "title": "openai/gpt-oss-20b" }, { "label": "License", "href": "https://huggingface.co/openai/gpt-oss-20b/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 9, "profile": { "General": 12.8, "Scientific": 6.2, "Coding": 19.4, "Agents": 0 }, "url": "https://artificialanalysis.ai/models/gpt-oss-20b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gpt-oss-20b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/gpt-oss-20b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gpt-oss-20b.md" }, { "slug": "qwen3-coder-flash-30b-a3b", "order": 39, "name": "Qwen3 Coder Flash (30B-A3B)", "baseTitle": "Qwen3 Coder Flash 30B-A3B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2025-07-31", "decoderType": "Sparse MoE", "attention": "GQA", "layerMix": "48 GQA", "scale": "30B total, 3.3B active (11% active)", "totalParams": 30000000000, "activeParams": 3300000000, "context": "256,000", "contextTokens": 256000, "license": "Apache License 2.0", "kvCacheText": "96 KiB", "kvCacheKiB": 96, "kvCacheLevel": "Moderate", "summary": "Coding-tuned Qwen model that keeps a straightforward grouped-query MoE stack instead of the newer hybrid-attention variants.", "keyDetail": "Uses 128 experts with 8 active per token and a native 256k context window for coding workloads.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-coder-flash-30b-a3b-mixture-of-experts.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-coder-flash-30b-a3b-mixture-of-experts.webp", "alt": "Figure 16: Qwen3 Coder Flash 30B-A3B", "width": 960, "height": 929 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A74-qwen3-coder-next-an-attention-hybrid-for-coding", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A74-qwen3-coder-next-an-attention-hybrid-for-coding", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/11_qwen3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/main/config.json", "title": "Qwen/Qwen3-Coder-30B-A3B-Instruct" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 9.6, "profile": { "Scientific": 1.9 }, "url": "https://artificialanalysis.ai/models/qwen3-coder-30b-a3b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-coder-flash-30b-a3b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-coder-flash-30b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-coder-flash-30b-a3b.md" }, { "slug": "glm-4-5-355b", "order": 17, "name": "GLM-4.5 (355B)", "baseTitle": "GLM-4.5 355B", "company": "Z.ai", "country": "China", "flag": "🇨🇳", "date": "2025-07-28", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm", "layerMix": "92 GQA", "scale": "355B total, 32B active (9% active)", "totalParams": 355000000000, "activeParams": 32000000000, "context": "128,000", "contextTokens": 128000, "license": "MIT License", "kvCacheText": "368 KiB", "kvCacheKiB": 368, "kvCacheLevel": "Very high", "summary": "Agent-oriented instruction/reasoning hybrid that borrows DeepSeek's dense-prefix MoE layout.", "keyDetail": "Starts with three dense layers before MoE routing, keeps a shared expert, and uses MTP during training.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/glm-4-5-355b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/glm-4-5-355b.webp", "alt": "Figure 34: GLM-4.5 355B", "width": 960, "height": 936 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A711-glm-45", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A711-glm-45", "title": null }, { "label": "config.json", "href": "https://huggingface.co/zai-org/GLM-4.5/blob/main/config.json", "title": "zai-org/GLM-4.5" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2508.06471", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 12.8, "profile": { "Scientific": 6.5 }, "url": "https://artificialanalysis.ai/models/glm-4.5" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-glm-4-5-355b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/glm-4-5-355b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/glm-4-5-355b.md" }, { "slug": "glm-4-5-air", "order": 51, "name": "GLM-4.5-Air (106B)", "baseTitle": "GLM-4.5-Air", "company": "Z.ai", "country": "China", "flag": "🇨🇳", "date": "2025-07-28", "decoderType": "Sparse MoE", "attention": "GQA", "layerMix": "46 GQA", "scale": "106B total, 12B active (11.3% active)", "totalParams": 106000000000, "activeParams": 12000000000, "context": "128,000", "contextTokens": 128000, "license": "MIT License", "kvCacheText": "184 KiB", "kvCacheKiB": 184, "kvCacheLevel": "High", "summary": "Compact GLM-4.5 companion that keeps the same agent-oriented sparse MoE recipe at a smaller serving footprint.", "keyDetail": "Shrinks the GLM-4.5 layout to 46 layers, keeps the dense warmup, and inherits the MTP training recipe.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/glm-4-5-air.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/glm-4-5-air.webp", "alt": "Figure 1: GLM-4.5-Air", "width": 960, "height": 936 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/zai-org/GLM-4.5-Air/blob/main/config.json", "title": "zai-org/GLM-4.5-Air" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2508.06471", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 11.1, "profile": { "Scientific": 3.5 }, "url": "https://artificialanalysis.ai/models/glm-4-5-air" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-glm-4-5-air", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/glm-4-5-air/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/glm-4-5-air.md" }, { "slug": "kimi-k2", "order": 16, "name": "Kimi K2 (1T)", "baseTitle": "Kimi K2", "company": "Moonshot AI", "country": "China", "flag": "🇨🇳", "date": "2025-07-10", "decoderType": "Sparse MoE", "attention": "MLA", "layerMix": "61 MLA", "scale": "1T total, 32B active (3.2% active)", "totalParams": 1000000000000, "activeParams": 32000000000, "context": "128,000", "contextTokens": 128000, "license": "Modified MIT License", "kvCacheText": "68.6 KiB", "kvCacheKiB": 68.6, "kvCacheLevel": "Low", "summary": "Trillion-parameter Moonshot model that essentially scales the DeepSeek V3 recipe upward.", "keyDetail": "More experts and fewer MLA heads than DeepSeek V3.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/kimi-k2-1-trillion.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/kimi-k2-1-trillion.webp", "alt": "Figure 25.1: Kimi K2", "width": 960, "height": 919 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A78-kimi-k2-and-kimi-k2-thinking", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A78-kimi-k2-and-kimi-k2-thinking", "title": null }, { "label": "config.json", "href": "https://huggingface.co/moonshotai/Kimi-K2-Base/blob/main/config.json", "title": "moonshotai/Kimi-K2-Base" }, { "label": "License", "href": "https://huggingface.co/moonshotai/Kimi-K2-Base/blame/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2507.20534", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 12.7, "profile": { "Scientific": 3.7 }, "url": "https://artificialanalysis.ai/models/kimi-k2" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-kimi-k2", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/kimi-k2/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/kimi-k2.md" }, { "slug": "smollm3-3b", "order": 15, "name": "SmolLM3 (3B)", "baseTitle": "SmolLM3 3B", "company": "Hugging Face", "country": "United States", "flag": "🇺🇸", "date": "2025-06-19", "decoderType": "Dense", "attention": "GQA with periodic NoPE layers", "layerMix": "36 GQA", "scale": "3B parameters", "totalParams": 3000000000, "activeParams": 3000000000, "context": "131,072", "contextTokens": 131072, "license": "Apache License 2.0", "kvCacheText": "72 KiB", "kvCacheKiB": 72, "kvCacheLevel": "Low", "summary": "Compact dense model that experiments with leaving out positional encodings in selected layers.", "keyDetail": "Every fourth layer omits RoPE to test a NoPE-style cadence.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/smollm3-3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/smollm3-3b.webp", "alt": "Figure 21: SmolLM3 3B", "width": 934, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A77-smollm3", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A77-smollm3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B-Base/blob/main/config.json", "title": "HuggingFaceTB/SmolLM3-3B-Base" }, { "label": "Tech report", "href": "https://huggingface.co/blog/smollm3", "title": null } ], "concepts": [ { "slug": "nope", "label": "NoPE", "name": "No Positional Embeddings", "url": "https://llm-atlas-7or.pages.dev/concepts/nope/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-smollm3-3b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/smollm3-3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/smollm3-3b.md" }, { "slug": "qwen3-0-6b", "order": 9, "name": "Qwen3 (0.6B)", "baseTitle": "Qwen3 0.6B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2025-04-28", "decoderType": "Dense", "attention": "GQA", "layerMix": "28 GQA", "scale": "0.6B parameters", "totalParams": 600000000, "activeParams": 600000000, "context": "32,768", "contextTokens": 32768, "license": "Apache License 2.0", "kvCacheText": "112 KiB", "kvCacheKiB": 112, "kvCacheLevel": "Moderate", "summary": "Tiny current-generation Qwen model that trades width for more depth and a low memory footprint.", "keyDetail": "Deeper than Llama 3.2 1B and unusually practical for local experiments and teaching.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-0-6b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-0-6b.webp", "alt": "Figure 18: Qwen3 0.6B", "width": 912, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A761-qwen3-dense", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A761-qwen3-dense", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3-0.6B-Base/blob/main/config.json", "title": "Qwen/Qwen3-0.6B-Base" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3-0.6B-Base/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 4.8, "profile": { "Scientific": 2.5 }, "url": "https://artificialanalysis.ai/models/qwen3-0.6b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-0-6b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-0-6b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-0-6b.md" }, { "slug": "qwen3-235b-a22b", "order": 10, "name": "Qwen3 (235B-A22B)", "baseTitle": "Qwen3 235B-A22B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2025-04-28", "decoderType": "Sparse MoE", "attention": "GQA with QK-Norm", "layerMix": "94 GQA", "scale": "235B total, 22B active (9.4% active)", "totalParams": 235000000000, "activeParams": 22000000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "188 KiB", "kvCacheKiB": 188, "kvCacheLevel": "High", "summary": "Large sparse Qwen variant that stays very close to DeepSeek V3 while removing the shared expert.", "keyDetail": "High-capacity MoE design optimized for serving efficiency without a shared expert.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-235b-a22b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-235b-a22b.webp", "alt": "Figure 19: Qwen3 235B-A22B", "width": 960, "height": 946 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A762-qwen3-moe", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A762-qwen3-moe", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/11_qwen3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3-235B-A22B/blob/main/config.json", "title": "Qwen/Qwen3-235B-A22B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3-235B-A22B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2505.09388", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 8.3, "profile": { "Scientific": 2.1 }, "url": "https://artificialanalysis.ai/models/qwen3-235b-a22b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-235b-a22b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-235b-a22b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-235b-a22b.md" }, { "slug": "qwen3-30b-a3b", "order": 11, "name": "Qwen3 (30B-A3B)", "baseTitle": "Qwen3 30B-A3B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2025-04-28", "decoderType": "Sparse MoE", "attention": "GQA", "layerMix": "48 GQA", "scale": "30B total, 3B active (10% active)", "totalParams": 30000000000, "activeParams": 3000000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "96 KiB", "kvCacheKiB": 96, "kvCacheLevel": "Moderate", "summary": "Small sparse Qwen3 model that sits close to GPT-OSS in active size but uses a deeper stack.", "keyDetail": "Deeper, narrower MoE alternative to GPT-OSS without a shared expert.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-30b-a3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-30b-a3b.webp", "alt": "Figure 27: Qwen3 30B-A3B", "width": 960, "height": 925 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A79-gpt-oss", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A79-gpt-oss", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3-30B-A3B/blob/main/config.json", "title": "Qwen/Qwen3-30B-A3B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3-30B-A3B/blob/main/LICENSE", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 6.6, "profile": { "Scientific": 2.3 }, "url": "https://artificialanalysis.ai/models/qwen3-30b-a3b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-30b-a3b", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-30b-a3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-30b-a3b.md" }, { "slug": "qwen3-32b", "order": 12, "name": "Qwen3 (32B)", "baseTitle": "Qwen3 32B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2025-04-28", "decoderType": "Dense", "attention": "GQA with QK-Norm", "layerMix": "64 GQA", "scale": "32B parameters", "totalParams": 32000000000, "activeParams": 32000000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "256 KiB", "kvCacheKiB": 256, "kvCacheLevel": "High", "summary": "Large dense Qwen3 model that serves as the clearest like-for-like comparison for OLMo 3 32B.", "keyDetail": "Reference dense Qwen stack with QK-Norm and 8 KV heads.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-32b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-32b.webp", "alt": "Figure 45: Qwen3 32B", "width": 918, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A715-olmo-3-thinking", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A715-olmo-3-thinking", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/11_qwen3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3-32B/blob/main/config.json", "title": "Qwen/Qwen3-32B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3-32B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2505.09388", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 7.3, "profile": {}, "url": "https://artificialanalysis.ai/models/qwen3-32b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-32b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-32b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-32b.md" }, { "slug": "qwen3-4b", "order": 13, "name": "Qwen3 (4B)", "baseTitle": "Qwen3 4B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2025-04-28", "decoderType": "Dense", "attention": "GQA with QK-Norm", "layerMix": "36 GQA", "scale": "4B parameters", "totalParams": 4000000000, "activeParams": 4000000000, "context": "32,768", "contextTokens": 32768, "license": "Apache License 2.0", "kvCacheText": "144 KiB", "kvCacheKiB": 144, "kvCacheLevel": "Moderate", "summary": "Mid-size dense Qwen3 model used here as a clean baseline against SmolLM3 and Tiny Aya.", "keyDetail": "Compact Qwen3 dense stack with QK-Norm and a 151k vocabulary.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-4b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-4b.webp", "alt": "Figure 21: Qwen3 4B", "width": 912, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A77-smollm3", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A77-smollm3", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/11_qwen3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3-4B/blob/main/config.json", "title": "Qwen/Qwen3-4B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3-4B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2505.09388", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 6.6, "profile": {}, "url": "https://artificialanalysis.ai/models/qwen3-4b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-4b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-4b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-4b.md" }, { "slug": "qwen3-8b", "order": 14, "name": "Qwen3 (8B)", "baseTitle": "Qwen3 8B", "company": "Alibaba Cloud", "country": "China", "flag": "🇨🇳", "date": "2025-04-28", "decoderType": "Dense", "attention": "GQA with QK-Norm", "layerMix": "36 GQA", "scale": "8B parameters", "totalParams": 8000000000, "activeParams": 8000000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "144 KiB", "kvCacheKiB": 144, "kvCacheLevel": "Moderate", "summary": "Dense Qwen3 baseline used here to show how little OLMo 3 changed the overall decoder recipe.", "keyDetail": "Reference Qwen3 dense stack with QK-Norm and 8 KV heads.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/qwen3-8b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/qwen3-8b.webp", "alt": "Figure 44: Qwen3 8B", "width": 916, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A715-olmo-3-thinking", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A715-olmo-3-thinking", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/11_qwen3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/Qwen/Qwen3-8B/blob/main/config.json", "title": "Qwen/Qwen3-8B" }, { "label": "License", "href": "https://huggingface.co/Qwen/Qwen3-8B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2505.09388", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 6, "profile": { "Scientific": 1 }, "url": "https://artificialanalysis.ai/models/qwen3-8b-instruct" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-qwen3-8b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/qwen3-8b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/qwen3-8b.md" }, { "slug": "llama-4-maverick", "order": 8, "name": "Llama 4 Maverick (400B)", "baseTitle": "Llama 4 Maverick", "company": "Meta", "country": "United States", "flag": "🇺🇸", "date": "2025-04-05", "decoderType": "Sparse MoE", "attention": "GQA", "layerMix": "36 chunked + 12 full GQA", "scale": "400B total, 17B active (4.3% active)", "totalParams": 400000000000, "activeParams": 17000000000, "context": "1,000,000", "contextTokens": 1000000, "license": "Llama 4 Community License Agreement", "kvCacheText": "192 KiB", "kvCacheKiB": 192, "kvCacheLevel": "High", "summary": "Meta's large MoE follows the DeepSeek V3 playbook but with a more conventional attention stack.", "keyDetail": "Alternates dense and MoE blocks and uses fewer, larger experts than DeepSeek V3.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/llama-4-maverick-400b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/llama-4-maverick-400b.webp", "alt": "Figure 17: Llama 4 Maverick", "width": 960, "height": 932 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A75-llama-4", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A75-llama-4", "title": null }, { "label": "config.json", "href": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct/blob/main/config.json", "title": "meta-llama/Llama-4-Maverick-17B-128E-Instruct" }, { "label": "Tech report", "href": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 9.3, "profile": { "General": 18.8, "Scientific": 2.5, "Coding": 15.9, "Agents": 0 }, "url": "https://artificialanalysis.ai/models/llama-4-maverick" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-llama-4-maverick", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/llama-4-maverick/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/llama-4-maverick.md" }, { "slug": "mistral-small-3-1-24b", "order": 7, "name": "Mistral Small 3.1 (24B)", "baseTitle": "Mistral Small 3.1 24B", "company": "Mistral AI", "country": "France", "flag": "🇫🇷", "date": "2025-03-18", "decoderType": "Dense", "attention": "Standard GQA", "layerMix": "40 GQA", "scale": "24B parameters", "totalParams": 24000000000, "activeParams": 24000000000, "context": "128,000", "contextTokens": 128000, "license": "Apache License 2.0", "kvCacheText": "160 KiB", "kvCacheKiB": 160, "kvCacheLevel": "Moderate", "summary": "Fast dense 24B model that drops the sliding-window setup used in older Mistral releases.", "keyDetail": "Latency-focused design with a smaller KV cache and fewer layers than Gemma 3 27B.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/mistral-3-1-small-24b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/mistral-3-1-small-24b.webp", "alt": "Figure 16: Mistral Small 3.1 24B", "width": 909, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A74-mistral-small-31", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A74-mistral-small-31", "title": null }, { "label": "config.json", "href": "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Base-2503/blob/main/config.json", "title": "mistralai/Mistral-Small-3.1-24B-Base-2503" }, { "label": "Tech report", "href": "https://mistral.ai/news/mistral-small-3-1", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 7.1, "profile": { "General": 12.8, "Scientific": 2.2, "Coding": 13.9, "Agents": 0.2 }, "url": "https://artificialanalysis.ai/models/mistral-small-3-1" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-mistral-small-3-1-24b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/mistral-small-3-1-24b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/mistral-small-3-1-24b.md" }, { "slug": "xlstm-7b", "order": 50, "name": "xLSTM (7B)", "baseTitle": "xLSTM 7B", "company": "NXAI", "country": "Austria", "flag": "🇦🇹", "date": "2025-03-17", "decoderType": "Recurrent", "attention": "No self-attention; mLSTM recurrent layers with matrix memory", "layerMix": "32 mLSTM", "scale": "7B parameters", "totalParams": 7000000000, "activeParams": 7000000000, "context": "No explicit limit", "contextTokens": null, "license": "NXAI Community License Agreement", "kvCacheText": "0 B", "kvCacheKiB": 0, "kvCacheLevel": "No cache", "summary": "Recurrent 7B language model that replaces self-attention with xLSTM blocks built around matrix memory.", "keyDetail": "Stateful recurrent architecture aimed at fast long-context inference without an explicit context window.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/xlstm-7b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/xlstm-7b.webp", "alt": "Figure 1: xLSTM 7B", "width": 960, "height": 880 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/NX-AI/xLSTM-7b/blob/main/config.json", "title": "NX-AI/xLSTM-7b" }, { "label": "License", "href": "https://huggingface.co/NX-AI/xLSTM-7b/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/abs/2503.13427", "title": null } ], "concepts": [], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-xlstm-7b", "family": "recurrent", "url": "https://llm-atlas-7or.pages.dev/models/xlstm-7b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/xlstm-7b.md" }, { "slug": "gemma-3-27b", "order": 6, "name": "Gemma 3 (27B)", "baseTitle": "Gemma 3 27B", "company": "Google", "country": "United States", "flag": "🇺🇸", "date": "2025-03-11", "decoderType": "Dense", "attention": "GQA with QK-Norm and 5:1 sliding-window/global attention", "layerMix": "52 sliding-window + 10 global", "scale": "27B parameters", "totalParams": 27000000000, "activeParams": 27000000000, "context": "128,000", "contextTokens": 128000, "license": "Gemma Terms of Use + Gemma Prohibited Use Policy", "kvCacheText": "496 KiB", "kvCacheKiB": 496, "kvCacheLevel": "Very high", "summary": "Gemma's flagship text stack leans on local attention more aggressively than Gemma 2.", "keyDetail": "Built around a 27B sweet spot with heavier local attention and a large 262k multilingual vocabulary.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gemma-3-27b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gemma-3-27b.webp", "alt": "Figure 16: Gemma 3 27B", "width": 909, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A74-mistral-small-31", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A74-mistral-small-31", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/12_gemma3", "title": null }, { "label": "config.json", "href": "https://huggingface.co/google/gemma-3-27b-it/blob/main/config.json", "title": "google/gemma-3-27b-it" }, { "label": "License", "href": "https://ai.google.dev/gemma/prohibited_use_policy", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2503.19786", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" }, { "slug": "swa", "label": "SWA", "name": "Sliding Window Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/swa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 4.9, "profile": { "General": 6.9, "Scientific": 2.2, "Coding": 11.6, "Agents": 0 }, "url": "https://artificialanalysis.ai/models/gemma-3-27b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gemma-3-27b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/gemma-3-27b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gemma-3-27b.md" }, { "slug": "deepseek-r1", "order": 5, "name": "DeepSeek R1 (671B)", "baseTitle": "DeepSeek R1", "company": "DeepSeek", "country": "China", "flag": "🇨🇳", "date": "2025-01-20", "decoderType": "Sparse MoE", "attention": "MLA", "layerMix": "61 MLA", "scale": "671B total, 37B active (5.5% active)", "totalParams": 671000000000, "activeParams": 37000000000, "context": "128,000", "contextTokens": 128000, "license": "MIT License", "kvCacheText": "68.6 KiB", "kvCacheKiB": 68.6, "kvCacheLevel": "Low", "summary": "Reasoning-tuned DeepSeek model built on the V3 architecture rather than a new base design.", "keyDetail": "Architecture matches DeepSeek V3, including the MTP-1 path; the main change is the reasoning-oriented training recipe.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/deepseek-v3-r1-671-billion.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/deepseek-v3-r1-671-billion.webp", "alt": "Figure 25.2: DeepSeek R1", "width": 960, "height": 938 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A78-kimi-k2-and-kimi-k2-thinking", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A78-kimi-k2-and-kimi-k2-thinking", "title": null }, { "label": "config.json", "href": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/main/config.json", "title": "deepseek-ai/DeepSeek-R1" }, { "label": "License", "href": "https://huggingface.co/deepseek-ai/DeepSeek-R1/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2501.12948", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 11.4, "profile": { "General": 22.2, "Scientific": 4.5, "Coding": 19.2, "Agents": 0 }, "url": "https://artificialanalysis.ai/models/deepseek-r1-0120" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-deepseek-r1", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/deepseek-r1/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/deepseek-r1.md" }, { "slug": "deepseek-v3", "order": 4, "name": "DeepSeek V3 (671B)", "baseTitle": "DeepSeek V3", "company": "DeepSeek", "country": "China", "flag": "🇨🇳", "date": "2024-12-26", "decoderType": "Sparse MoE", "attention": "MLA", "layerMix": "61 MLA", "scale": "671B total, 37B active (5.5% active)", "totalParams": 671000000000, "activeParams": 37000000000, "context": "128,000", "contextTokens": 128000, "license": "DeepSeek License Agreement v1.0", "kvCacheText": "68.6 KiB", "kvCacheKiB": 68.6, "kvCacheLevel": "Low", "summary": "DeepSeek's flagship template kicked off the recent wave of large open MoE models.", "keyDetail": "Uses a dense prefix, shared expert, and MTP-1 objective to keep a very large model practical.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/deepseek-v3-r1-671-billion.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/deepseek-v3-r1-671-billion.webp", "alt": "Figure 17: DeepSeek V3", "width": 960, "height": 938 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A75-llama-4", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A75-llama-4", "title": null }, { "label": "config.json", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V3/blob/main/config.json", "title": "deepseek-ai/DeepSeek-V3" }, { "label": "License", "href": "https://huggingface.co/deepseek-ai/DeepSeek-V3/blame/main/LICENSE-MODEL", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2412.19437", "title": null } ], "concepts": [ { "slug": "mla", "label": "MLA", "name": "Multi-Head Latent Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mla/" }, { "slug": "moe", "label": "MoE", "name": "Mixture of Experts", "url": "https://llm-atlas-7or.pages.dev/concepts/moe/" }, { "slug": "mtp", "label": "MTP", "name": "Multi-Token Prediction", "url": "https://llm-atlas-7or.pages.dev/concepts/mtp/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 8.5, "profile": { "General": 15.3, "Scientific": 1.5, "Coding": 17.9, "Agents": 0.1 }, "url": "https://artificialanalysis.ai/models/deepseek-v3" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-deepseek-v3", "family": "moe", "url": "https://llm-atlas-7or.pages.dev/models/deepseek-v3/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/deepseek-v3.md" }, { "slug": "phi-4", "order": 49, "name": "Phi-4 (14B)", "baseTitle": "Phi-4", "company": "Microsoft", "country": "United States", "flag": "🇺🇸", "date": "2024-12-12", "decoderType": "Dense", "attention": "GQA with RoPE", "layerMix": "40 GQA", "scale": "14B parameters", "totalParams": 14000000000, "activeParams": 14000000000, "context": "16,384", "contextTokens": 16384, "license": "MIT License", "kvCacheText": "200 KiB", "kvCacheKiB": 200, "kvCacheLevel": "High", "summary": "Microsoft's 14B dense Phi refresh stays close to Phi-3-medium but swaps its sliding-window attention for full-context GQA and a larger tokenizer.", "keyDetail": "Classic pre-norm RMSNorm stack with GQA, 40 heads, 10 KV heads, and a 100,352-token vocabulary.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/phi-4.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/phi-4.webp", "alt": "Figure 1: Phi-4", "width": 917, "height": 960 }, "articleUrl": null, "links": [ { "label": "config.json", "href": "https://huggingface.co/microsoft/phi-4/blob/main/config.json", "title": "microsoft/phi-4" }, { "label": "License", "href": "https://huggingface.co/microsoft/phi-4/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2412.08905", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 5.9, "profile": { "Scientific": 1.9 }, "url": "https://artificialanalysis.ai/models/phi-4" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-phi-4", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/phi-4/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/phi-4.md" }, { "slug": "olmo-2-7b", "order": 3, "name": "OLMo 2 (7B)", "baseTitle": "OLMo 2 7B", "company": "Ai2", "country": "United States", "flag": "🇺🇸", "date": "2024-11-25", "decoderType": "Dense", "attention": "MHA with QK-Norm", "layerMix": "32 MHA", "scale": "7B parameters", "totalParams": 7000000000, "activeParams": 7000000000, "context": "4,096", "contextTokens": 4096, "license": "Apache License 2.0", "kvCacheText": "512 KiB", "kvCacheKiB": 512, "kvCacheLevel": "Very high", "summary": "Transparent dense model that keeps classic MHA and pushes normalization changes for training stability.", "keyDetail": "Uses inside-residual post-norm instead of the usual pre-norm layout.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/olmo-2-7b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/olmo-2-7b.webp", "alt": "Figure 10: OLMo 2 7B", "width": 925, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A723-olmo-2-summary", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A723-olmo-2-summary", "title": null }, { "label": "config.json", "href": "https://huggingface.co/allenai/OLMo-2-1124-7B-Instruct/blob/main/config.json", "title": "allenai/OLMo-2-1124-7B-Instruct" }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2501.00656", "title": null } ], "concepts": [ { "slug": "qk-norm", "label": "QK-Norm", "name": "QK-Norm", "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/" }, { "slug": "mha", "label": "MHA", "name": "Multi-Head Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mha/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 5.6, "profile": {}, "url": "https://artificialanalysis.ai/models/olmo-2-7b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-olmo-2-7b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/olmo-2-7b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/olmo-2-7b.md" }, { "slug": "llama-3-2-1b", "order": 2, "name": "Llama 3.2 (1B)", "baseTitle": "Llama 3.2 1B", "company": "Meta", "country": "United States", "flag": "🇺🇸", "date": "2024-09-25", "decoderType": "Dense", "attention": "GQA", "layerMix": "16 GQA", "scale": "1B parameters", "totalParams": 1000000000, "activeParams": 1000000000, "context": "128,000", "contextTokens": 128000, "license": "Llama Community License Agreement (variant-specific)", "kvCacheText": "32 KiB", "kvCacheKiB": 32, "kvCacheLevel": "Low", "summary": "Small dense Llama baseline in the Qwen comparison, with fewer layers but more width.", "keyDetail": "Wider architecture with more heads than Qwen3 0.6B.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/llama-3-2-1b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/llama-3-2-1b.webp", "alt": "Figure 18: Llama 3.2 1B", "width": 916, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A761-qwen3-dense", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A761-qwen3-dense", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/07_gpt_to_llama", "title": null }, { "label": "License", "href": "https://huggingface.co/meta-llama/Llama-3.2-1B/blob/main/LICENSE.txt", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 4.8, "profile": { "Scientific": 2.7 }, "url": "https://artificialanalysis.ai/models/llama-3-2-instruct-1b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-llama-3-2-1b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/llama-3-2-1b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/llama-3-2-1b.md" }, { "slug": "llama-3-2-3b", "order": 38, "name": "Llama 3.2 (3B)", "baseTitle": "Llama 3.2 3B", "company": "Meta", "country": "United States", "flag": "🇺🇸", "date": "2024-09-25", "decoderType": "Dense", "attention": "GQA", "layerMix": "28 GQA", "scale": "3B parameters", "totalParams": 3000000000, "activeParams": 3000000000, "context": "128,000", "contextTokens": 128000, "license": "Llama 3.2 Community License Agreement", "kvCacheText": "112 KiB", "kvCacheKiB": 112, "kvCacheLevel": "Moderate", "summary": "Small Llama baseline used to show how closely Nanbeige follows the mainstream compact decoder recipe.", "keyDetail": "Reference small-model Llama architecture with tied embeddings.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/llama-3-2-3b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/llama-3-2-3b.webp", "alt": "Figure 29: Llama 3.2 3B", "width": 916, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A77-nanbeige-41-3b-a-strong-llama-3-successor", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/a-dream-of-spring-for-open-weight#%C2%A77-nanbeige-41-3b-a-strong-llama-3-successor", "title": null }, { "label": "License", "href": "https://huggingface.co/meta-llama/Llama-3.2-3B/blob/main/LICENSE.txt", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 5.7, "profile": {}, "url": "https://artificialanalysis.ai/models/llama-3-2-instruct-3b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-llama-3-2-3b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/llama-3-2-3b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/llama-3-2-3b.md" }, { "slug": "llama-3-8b", "order": 1, "name": "Llama 3 (8B)", "baseTitle": "Llama 3 8B", "company": "Meta", "country": "United States", "flag": "🇺🇸", "date": "2024-04-18", "decoderType": "Dense", "attention": "GQA with RoPE", "layerMix": "32 GQA", "scale": "8B parameters", "totalParams": 8000000000, "activeParams": 8000000000, "context": "8,192", "contextTokens": 8192, "license": "Llama 3 Community License Agreement", "kvCacheText": "128 KiB", "kvCacheKiB": 128, "kvCacheLevel": "Moderate", "summary": "Reference dense Llama stack used to contrast OLMo 2's normalization and attention choices.", "keyDetail": "Pre-norm baseline; wider than OLMo 2 at a similar scale.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/llama-3-8b.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/llama-3-8b.webp", "alt": "Figure 10: Llama 3 8B", "width": 914, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A723-olmo-2-summary", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison#%C2%A723-olmo-2-summary", "title": null }, { "label": "From scratch", "href": "https://github.com/rasbt/LLMs-from-scratch/tree/main/ch05/07_gpt_to_llama", "title": null }, { "label": "config.json", "href": "https://huggingface.co/meta-llama/Meta-Llama-3-8B/blob/main/config.json", "title": "meta-llama/Meta-Llama-3-8B" }, { "label": "License", "href": "https://huggingface.co/meta-llama/Meta-Llama-3-8B/blob/main/LICENSE", "title": null }, { "label": "Tech report", "href": "https://arxiv.org/pdf/2407.21783", "title": null } ], "concepts": [ { "slug": "gqa", "label": "GQA", "name": "Grouped-Query Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/" } ], "aai": { "v4": { "label": "AA Intelligence Index v4.3.2", "total": 4.8, "profile": { "Scientific": 2.5 }, "url": "https://artificialanalysis.ai/models/llama-3-instruct-8b" } }, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-llama-3-8b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/llama-3-8b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/llama-3-8b.md" }, { "slug": "gpt-2-xl-1-5b", "order": 0, "name": "GPT-2 XL (1.5B)", "baseTitle": "GPT-2 XL 1.5B", "company": "OpenAI", "country": "United States", "flag": "🇺🇸", "date": "2019-11-05", "decoderType": "Dense", "attention": "MHA with learned absolute positional embeddings", "layerMix": "48 MHA", "scale": "1.5B parameters", "totalParams": 1500000000, "activeParams": 1500000000, "context": "1,024", "contextTokens": 1024, "license": "OpenAI \"Modified MIT\" license", "kvCacheText": "300 KiB", "kvCacheKiB": 300, "kvCacheLevel": "High", "summary": "Late-2019 dense baseline included here as a reference point for how much decoder stacks have changed since GPT-2.", "keyDetail": "Classic GPT-2 recipe with dropout, GELU, LayerNorm, and full multi-head attention.", "image": { "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/gpt-2-xl.webp", "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/gpt-2-xl.webp", "alt": "Figure 2: GPT-2 XL 1.5B", "width": 882, "height": 960 }, "articleUrl": "https://magazine.sebastianraschka.com/p/from-gpt-2-to-gpt-oss-analyzing-the#%C2%A72-coming-from-gpt-2", "links": [ { "label": "View in article", "href": "https://magazine.sebastianraschka.com/p/from-gpt-2-to-gpt-oss-analyzing-the#%C2%A72-coming-from-gpt-2", "title": null }, { "label": "config.json", "href": "https://huggingface.co/openai-community/gpt2-xl/blob/main/config.json", "title": "openai-community/gpt2-xl" }, { "label": "Tech report", "href": "https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf", "title": null } ], "concepts": [ { "slug": "mha", "label": "MHA", "name": "Multi-Head Attention", "url": "https://llm-atlas-7or.pages.dev/concepts/mha/" } ], "aai": null, "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-gpt-2-xl-1-5b", "family": "dense", "url": "https://llm-atlas-7or.pages.dev/models/gpt-2-xl-1-5b/", "markdownUrl": "https://llm-atlas-7or.pages.dev/models/gpt-2-xl-1-5b.md" } ] }