{
  "slug": "dory1-0-4b",
  "order": 113,
  "name": "Dory1.0 (4B)",
  "baseTitle": "Dory1.0 4B",
  "company": "NVIDIA",
  "country": "United States",
  "flag": "🇺🇸",
  "date": "2026-10-16",
  "decoderType": "Dense",
  "attention": "Gated GQA; 3:1 sliding-window/full, RoPE",
  "layerMix": "8 input + 20 shared × 2 passes + 8 output (56 applications)",
  "scale": "4B parameters",
  "totalParams": 4000000000,
  "activeParams": 4000000000,
  "context": "262,144",
  "contextTokens": 262144,
  "license": "OpenMDW 1.1",
  "kvCacheText": "224 KiB",
  "kvCacheKiB": 224,
  "kvCacheLevel": "High",
  "summary": "NVIDIA's dense looped transformer applies a shared 20-block core twice between 8-block input and output stacks, using normalized GPT (nGPT) residual updates.",
  "keyDetail": "Reinjects the input-stack output before each recurrent pass. The 36 distinct blocks give 56 block applications by default, with L2-normalized residual updates and a 1,024-token attention window.",
  "image": {
    "full": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/dory-1-0-4b.webp",
    "thumb": "https://sebastianraschka.com/llm-architecture-gallery/images/architectures/thumbnails/dory-1-0-4b.webp",
    "alt": "Figure 1: Dory1.0 4B",
    "width": 960,
    "height": 918
  },
  "articleUrl": null,
  "links": [
    {
      "label": "config.json",
      "href": "https://huggingface.co/nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16/blob/main/config.json",
      "title": "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-BF16"
    },
    {
      "label": "License",
      "href": "https://openmdw.ai/license/1-1/",
      "title": null
    }
  ],
  "concepts": [
    {
      "slug": "looped-depth-sharing",
      "label": "Looped Transformer",
      "name": "Looped Transformer",
      "url": "https://llm-atlas-7or.pages.dev/concepts/looped-depth-sharing/"
    },
    {
      "slug": "gqa",
      "label": "GQA",
      "name": "Grouped-Query Attention",
      "url": "https://llm-atlas-7or.pages.dev/concepts/gqa/"
    },
    {
      "slug": "swa",
      "label": "SWA",
      "name": "Sliding Window Attention",
      "url": "https://llm-atlas-7or.pages.dev/concepts/swa/"
    },
    {
      "slug": "gated-attention",
      "label": "Gated Attention",
      "name": "Gated Attention",
      "url": "https://llm-atlas-7or.pages.dev/concepts/gated-attention/"
    },
    {
      "slug": "qk-norm",
      "label": "QK-Norm",
      "name": "QK-Norm",
      "url": "https://llm-atlas-7or.pages.dev/concepts/qk-norm/"
    }
  ],
  "aai": null,
  "sourceUrl": "https://sebastianraschka.com/llm-architecture-gallery/#card-dory1-0-4b",
  "family": "dense",
  "url": "https://llm-atlas-7or.pages.dev/models/dory1-0-4b/",
  "markdownUrl": "https://llm-atlas-7or.pages.dev/models/dory1-0-4b.md"
}