{ "count": 489, "tracks": [ { "id": "model", "name": "How a model works", "description": "Tokens, the transformer block and how text is generated." }, { "id": "serving", "name": "Serving at scale", "description": "Inference internals, batching, parallelism, MoE and the GPU underneath." }, { "id": "agents", "name": "Agents", "description": "The agent loop, tool calling and managing context and memory." }, { "id": "production", "name": "Production systems", "description": "Backends, security, sandboxing and observability." }, { "id": "research", "name": "Training, tools & research", "description": "How models are trained and edited, the tool stack, and open problems." } ], "topics": [ { "n": 1, "slug": "fundamentals", "name": "LLM fundamentals", "tagline": "From raw text to a probability for every possible next token.", "track": "model", "intro": [ "A language model never sees letters. Text is cut into tokens, each token becomes an integer, and each integer becomes a vector. Those vectors flow through a stack of layers, and at the end a single vector is turned into one score for every token in the vocabulary.", "Generation is just that process on repeat: pick a token from the scores, append it, run again. Almost everything else in this guide, from KV caches to agent runtimes, exists to make that loop cheaper, faster or more useful." ], "figure": { "title": "Text in, next token out", "description": "Type anything. The text is split into subword tokens, mapped to IDs and embeddings, and the model scores candidates for the next token. Press Generate to append the top pick and run the loop again. Token IDs and scores here are illustrative, not from a real tokenizer." }, "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/fundamentals.md", "terms": [ { "name": "Tokens", "slug": "tokens", "def": "The units a language model reads and writes. A token is usually a word, a piece of a word, a punctuation mark or a byte sequence. In English one token is on average about three quarters of a word.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#tokens" }, { "name": "Tokenization", "slug": "tokenization", "def": "Splitting raw text into tokens before the model sees it. Most LLMs use subword schemes such as byte-pair encoding (BPE), which keep frequent words whole and break rare words into reusable pieces.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#tokenization" }, { "name": "Tokenizer", "slug": "tokenizer", "def": "The component that converts text to tokens and back. It is built before the model is trained and ships with the weights; swapping it breaks the model, because every token ID would suddenly mean something else.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#tokenizer" }, { "name": "Token IDs", "slug": "token-ids", "def": "The integer index of each token in the vocabulary. A word like \"Hello\" might become 9906. The network only ever sees these numbers, never characters.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#token-ids" }, { "name": "Vocabulary", "slug": "vocabulary", "def": "The fixed list of every token the tokenizer can produce, including special tokens. Each entry has exactly one ID and one row in the embedding table.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#vocabulary" }, { "name": "Vocabulary size", "slug": "vocabulary-size", "def": "How many entries the vocabulary has, typically 32K to 256K in current models. A larger vocabulary packs text into fewer tokens, especially code and non-English text, but grows the embedding table and the LM head.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#vocabulary-size" }, { "name": "Token embeddings", "slug": "token-embeddings", "def": "Learned vectors, one per vocabulary entry. Looking up a token ID returns its row of the embedding matrix, which turns a discrete symbol into numbers the network can compute with.", "concepts": [ "per-layer-embeddings" ], "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#token-embeddings" }, { "name": "Embedding dimension", "slug": "embedding-dimension", "def": "The length of each embedding vector, also called the hidden size or d_model, for example 4,096 or 7,168. It sets the width of the residual stream that runs through every layer.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#embedding-dimension" }, { "name": "Hidden states", "slug": "hidden-states", "def": "The vectors carried between layers, one per token position. Each block reads them, adds an update and passes them on, so they gradually shift from \"which token is this\" to \"what should come next\".", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#hidden-states" }, { "name": "Final hidden state", "slug": "final-hidden-state", "def": "The hidden state at the last position after the final layer and final norm. It is the only vector the LM head needs to score the next token.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#final-hidden-state" }, { "name": "Positional embeddings", "slug": "positional-embeddings", "def": "Information about where each token sits in the sequence. Attention by itself ignores order, so models either add learned position vectors to the embeddings or, more commonly today, rotate queries and keys by position (RoPE).", "concepts": [ "nope" ], "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#positional-embeddings" }, { "name": "Autoregressive generation", "slug": "autoregressive-generation", "def": "Producing text one token at a time, where each new token is appended to the input and fed back in to predict the next. It is why generation is sequential and why output length drives latency.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#autoregressive-generation" }, { "name": "Forward pass", "slug": "forward-pass", "def": "One run of inputs through the network, from embeddings to logits. Training pairs every forward pass with a backward pass; inference only does forward passes, one per generated token after the prompt.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#forward-pass" }, { "name": "Model parameters", "slug": "model-parameters", "def": "All the learned numbers in a network: embedding tables, attention and FFN weight matrices, norm scales. A \"70B\" model has roughly seventy billion of them.", "href": "/models/?sort=largest", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#model-parameters" }, { "name": "Model weights", "slug": "model-weights", "def": "The stored values of those parameters, usually as files in bf16, fp8 or a quantized format. \"Open weights\" means these files can be downloaded, even if the training data and code cannot.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#model-weights" }, { "name": "Weight matrices", "slug": "weight-matrices", "def": "The 2D arrays that hold most parameters: query, key, value and output projections, and the FFN up, gate and down projections. Nearly all compute in an LLM is multiplying activations by them.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#weight-matrices" }, { "name": "Matrix multiplication", "slug": "matrix-multiplication", "def": "The core operation of every layer (GEMM): activations times weights. GPUs are built around doing it fast, and how quickly they can do it and feed it with data decides model speed.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#matrix-multiplication" }, { "name": "LM head", "slug": "lm-head", "def": "The final linear layer that maps the final hidden state to one score per vocabulary token. Some models tie it to the embedding matrix to save parameters.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#lm-head" }, { "name": "Output projection", "slug": "output-projection", "def": "A linear layer that maps a sublayer's result back to the hidden size, such as attention's W_O after the heads are concatenated. The LM head is sometimes called the output projection of the whole model.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#output-projection" }, { "name": "Logits", "slug": "logits", "def": "The raw, unnormalised scores the LM head produces, one per vocabulary token. Higher means more likely, and they can be any real number.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#logits" }, { "name": "Softmax", "slug": "softmax", "def": "The function that turns scores into probabilities: exponentiate each score, then divide by the sum. It is used on the logits at the output and on attention scores inside every attention layer.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#softmax" }, { "name": "Probability distribution", "slug": "probability-distribution", "def": "The softmax output over the whole vocabulary: non-negative values that add up to 1. Decoding strategies decide how to pick one token from it.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#probability-distribution" }, { "name": "Next-token prediction", "slug": "next-token-prediction", "def": "The training objective of LLMs: given all previous tokens, predict the next one. Repeated over trillions of tokens, this one task is enough to learn grammar, facts and patterns of reasoning.", "concepts": [ "mtp" ], "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#next-token-prediction" }, { "name": "Next-token generation", "slug": "next-token-generation", "def": "Using a trained model to produce the next token at inference time: run a forward pass, read the distribution, pick a token with a decoding strategy, append it.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#next-token-generation" }, { "name": "Detokenization", "slug": "detokenization", "def": "Turning generated token IDs back into text. Byte-level tokenizers can split one character (an emoji, say) across tokens, so streaming output is buffered until the bytes form valid text.", "url": "https://llm-atlas-7or.pages.dev/learn/fundamentals/#detokenization" } ] }, { "n": 2, "slug": "transformer", "name": "Transformer architecture", "tagline": "The repeated block that almost every modern LLM is built from.", "track": "model", "intro": [ "A decoder-only transformer is a tall stack of identical blocks. Each block has two sublayers: attention, where tokens look at each other, and a feed-forward network, where each token is processed on its own. Both add their result back onto a shared residual stream.", "The ideas in the Concepts section, such as GQA, MLA, sliding windows and MoE, are mostly changes to one of these two sublayers. Once the basic block makes sense, every architecture diagram in the gallery reads as a set of edits to it." ], "figure": { "title": "Who attends to whom", "description": "Each row is a query token, each column a key it can look at. Pick a head to see a different learned pattern, and toggle the causal mask to see why a decoder can only look backwards. Hover or tap a row to read its weights. Patterns are illustrative." }, "url": "https://llm-atlas-7or.pages.dev/learn/transformer/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/transformer.md", "terms": [ { "name": "Decoder-only Transformer", "slug": "decoder-only-transformer", "def": "The architecture of nearly every current LLM: a stack of transformer blocks with causal self-attention and no separate encoder. It reads a sequence and predicts the next token at every position.", "concepts": [ "mha" ], "href": "/models/", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#decoder-only-transformer" }, { "name": "Transformer blocks", "slug": "transformer-blocks", "def": "The repeated unit of the network, often just called layers. A modern pre-norm block runs RMSNorm, attention, a residual add, RMSNorm, FFN and another residual add. Models stack anywhere from about 24 to over 100 of them.", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#transformer-blocks" }, { "name": "Self-attention", "slug": "self-attention", "def": "The operation that lets each token gather information from other tokens in the same sequence. Every position builds a query, compares it with every key, and takes a weighted average of the values.", "concepts": [ "mha" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#self-attention" }, { "name": "Attention layers", "slug": "attention-layers", "def": "The attention sublayer of each block: Q, K and V projections, the attention computation and the output projection. Variants such as GQA, MLA and sliding-window attention change this layer to save memory or compute.", "concepts": [ "gqa", "mla", "swa" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#attention-layers" }, { "name": "Query (Q)", "slug": "query-q", "def": "A vector each token produces that asks \"what am I looking for?\". Its dot product with every key decides how much that token attends to each position.", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#query-q" }, { "name": "Key (K)", "slug": "key-k", "def": "A vector each token produces that advertises \"what I contain\". During generation the keys of past tokens are stored in the KV cache.", "concepts": [ "gqa" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#key-k" }, { "name": "Value (V)", "slug": "value-v", "def": "The vector a token hands over when it is attended to: the information that actually moves. The attention output is a weighted sum of values.", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#value-v" }, { "name": "Attention masks", "slug": "attention-masks", "def": "A matrix added to the attention scores that blocks certain pairs, usually by setting them to minus infinity before the softmax. Used for causality, padding and packing several documents into one sequence.", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#attention-masks" }, { "name": "Causal attention", "slug": "causal-attention", "def": "Attention where a token may only look at itself and earlier tokens, enforced with an upper-triangular mask. It lets the model learn from every position of a sequence in one pass without seeing the answer.", "concepts": [ "swa" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#causal-attention" }, { "name": "Multi-head attention", "slug": "multi-head-attention", "def": "Several attention operations (heads) run in parallel, each with its own Q, K and V projections over a slice of the hidden size, then their results are concatenated. Different heads learn different patterns.", "concepts": [ "mha", "gqa" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#multi-head-attention" }, { "name": "Rotary Positional Embeddings (RoPE)", "slug": "rotary-positional-embeddings-rope", "def": "Encodes position by rotating pairs of query and key dimensions by an angle that grows with the token's position. The attention score then depends on relative distance, and the rotation frequencies can be rescaled to extend the context.", "concepts": [ "nope" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#rotary-positional-embeddings-rope" }, { "name": "RMSNorm", "slug": "rmsnorm", "def": "A normalisation that divides each vector by its root-mean-square and multiplies by a learned scale. It is cheaper than LayerNorm (no mean, no bias) and is the default in modern LLMs.", "concepts": [ "qk-norm" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#rmsnorm" }, { "name": "Layer normalization", "slug": "layer-normalization", "def": "Normalises each token's vector to zero mean and unit variance, then applies a learned scale and shift. Used in the original Transformer and GPT-2; most new models use RMSNorm instead.", "concepts": [ "qk-norm", "polynorm" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#layer-normalization" }, { "name": "Feed-forward network (FFN)", "slug": "feed-forward-network-ffn", "def": "The per-token sublayer after attention: widen the vector (often 3 to 4 times), apply a nonlinearity, project back. In modern models it is a gated SwiGLU and holds most of the parameters.", "concepts": [ "moe" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#feed-forward-network-ffn" }, { "name": "Multi-layer perceptron (MLP)", "slug": "multi-layer-perceptron-mlp", "def": "Another name for the FFN sublayer: a small fully connected network applied to each token independently. In MoE models every expert is one MLP.", "concepts": [ "moe" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#multi-layer-perceptron-mlp" }, { "name": "Residual connections", "slug": "residual-connections", "def": "Skip paths that add each sublayer's input to its output, so layers write updates onto a shared residual stream instead of replacing it. They keep signals and gradients flowing through very deep stacks.", "concepts": [ "gated-residuals", "mhc", "attention-residuals" ], "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#residual-connections" }, { "name": "Residual addition", "slug": "residual-addition", "def": "The add operation x + f(x) at the end of each sublayer. It is cheap, and it is the reason every layer must keep the same hidden size.", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#residual-addition" }, { "name": "Final normalization", "slug": "final-normalization", "def": "One last norm on the hidden states after the final block and before the LM head, which keeps the scale of the residual stream stable for the output layer.", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#final-normalization" }, { "name": "Linear layers", "slug": "linear-layers", "def": "Layers that multiply their input by a weight matrix, optionally adding a bias. Q, K, V and O projections, FFN projections, MoE routers and the LM head are all linear layers; modern LLMs mostly drop the biases.", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#linear-layers" }, { "name": "Model architecture", "slug": "model-architecture", "def": "The full specification of a network: layer count, hidden size, heads, attention type, FFN or MoE layout, norms, positions, vocabulary and context length. The gallery compares these choices across 100+ releases.", "href": "/compare/", "url": "https://llm-atlas-7or.pages.dev/learn/transformer/#model-architecture" } ] }, { "n": 3, "slug": "decoding", "name": "Decoding & sampling", "tagline": "How a probability distribution becomes an actual word.", "track": "model", "intro": [ "The model hands back a probability for every token in its vocabulary. Decoding is the policy that picks one. Always taking the top token gives repeatable but often flat text; sampling adds variety, and settings like temperature, top-k and top-p control how much.", "The same step is where structure is enforced. By masking every token that would break a grammar or JSON schema, an engine can guarantee valid output instead of hoping for it. Tool calls in agents depend on exactly this." ], "figure": { "title": "Sampling playground", "description": "Candidates for the word after \"The weather today is\". Move the sliders and watch the distribution reshape: temperature sharpens or flattens it, top-k and top-p cut the tail. Sample a few times to see how often each word wins." }, "url": "https://llm-atlas-7or.pages.dev/learn/decoding/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/decoding.md", "terms": [ { "name": "Decoding", "slug": "decoding", "def": "The strategy that turns the model's probability distribution into an actual token at each step, repeated until a stopping condition is met.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#decoding" }, { "name": "Sampling", "slug": "sampling", "def": "Picking the next token at random in proportion to its probability, instead of always taking the most likely one. It gives varied, more natural text at the cost of determinism.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#sampling" }, { "name": "Sampling settings", "slug": "sampling-settings", "def": "The knobs exposed by an API or engine: temperature, top-k, top-p, min-p, penalties, seed and maximum tokens. Code and data extraction usually want little randomness; brainstorming wants more.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#sampling-settings" }, { "name": "Temperature", "slug": "temperature", "def": "A number the logits are divided by before the softmax. Below 1 sharpens the distribution toward the top token, above 1 flattens it, and 0 is treated as greedy decoding.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#temperature" }, { "name": "Temperature scaling", "slug": "temperature-scaling", "def": "The operation logits ÷ T itself. It changes how far apart the token probabilities are without changing their order.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#temperature-scaling" }, { "name": "Top-k sampling", "slug": "top-k-sampling", "def": "Keep only the k most likely tokens, renormalise and sample among them. It cuts off the long tail of unlikely tokens, but uses the same k whether the model is confident or unsure.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#top-k-sampling" }, { "name": "Top-p / nucleus sampling", "slug": "top-p-nucleus-sampling", "def": "Keep the smallest set of top tokens whose probabilities add up to p (say 0.9), then sample. The set shrinks when the model is confident and grows when it is unsure.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#top-p-nucleus-sampling" }, { "name": "Greedy decoding", "slug": "greedy-decoding", "def": "Always choose the single most likely token. Deterministic and cheap, but prone to repetition and bland text in open-ended writing.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#greedy-decoding" }, { "name": "Argmax", "slug": "argmax", "def": "The operation that returns the position of the largest value. Greedy decoding is an argmax over the logits at every step.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#argmax" }, { "name": "Repetition penalty", "slug": "repetition-penalty", "def": "Lowers the scores of tokens that already appear in the context, for example by dividing their positive logits by a factor, to discourage loops.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#repetition-penalty" }, { "name": "Frequency penalty", "slug": "frequency-penalty", "def": "Subtracts an amount proportional to how often a token has already been generated, so heavily repeated tokens are pushed down more. The related presence penalty applies a flat amount once a token has appeared at all.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#frequency-penalty" }, { "name": "Logit processing", "slug": "logit-processing", "def": "Any step that changes the logits between the LM head and the final pick: temperature, penalties, logit bias, masks. Engines run them as a chain of logit processors.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#logit-processing" }, { "name": "Logit masking", "slug": "logit-masking", "def": "Setting some logits to minus infinity so those tokens can never be chosen. It is the mechanism behind constrained decoding and banned tokens.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#logit-masking" }, { "name": "Constrained decoding", "slug": "constrained-decoding", "def": "Limiting generation so the output always matches a formal rule, such as a regex, a grammar or a fixed list of options, by masking every token that would break it at each step.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#constrained-decoding" }, { "name": "Structured generation", "slug": "structured-generation", "def": "Producing output in a guaranteed shape, such as JSON, XML or a function call, rather than asking nicely. Usually built on constrained decoding driven by a grammar or schema.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#structured-generation" }, { "name": "JSON-constrained decoding", "slug": "json-constrained-decoding", "def": "Constrained decoding whose grammar is compiled from a JSON Schema, so the output is always parseable JSON with the required fields. Used for tool calls and data extraction.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#json-constrained-decoding" }, { "name": "Token selection", "slug": "token-selection", "def": "The last step of each decoding iteration: choose one token ID from the processed distribution, by argmax or by sampling.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#token-selection" }, { "name": "Stopping conditions", "slug": "stopping-conditions", "def": "The rules that end generation: an end-of-sequence token, a stop string, the maximum token count, or the end of a tool call.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#stopping-conditions" }, { "name": "End-of-sequence token", "slug": "end-of-sequence-token", "def": "A special token such as <|endoftext|> or <|im_end|> that the model learns to emit when its answer is complete. Generation stops when it is picked.", "url": "https://llm-atlas-7or.pages.dev/learn/decoding/#end-of-sequence-token" } ] }, { "n": 4, "slug": "inference", "name": "LLM inference internals", "tagline": "Why the first token is slow and the rest are fast.", "track": "serving", "intro": [ "Every request has two very different halves. Prefill reads the whole prompt in one big parallel pass and fills the KV cache. Decode then produces one token per step, and each step has to read all the model weights plus the cache for very little arithmetic.", "That split explains most serving behaviour. Prefill is limited by compute and sets time to first token; decode is limited by memory bandwidth and sets streaming speed. Caching, speculative decoding and careful scheduling all attack one half or the other." ], "figure": { "title": "Prefill, then decode", "description": "The prompt's tokens are processed together in one pass, filling the KV cache. After that, each step adds one token. Turn the cache off to see why it exists: without it every step recomputes the whole sequence, and the work per token keeps growing." }, "url": "https://llm-atlas-7or.pages.dev/learn/inference/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/inference.md", "terms": [ { "name": "Inference", "slug": "inference", "def": "Running a trained model to get outputs. For LLMs it splits into processing the prompt, then generating tokens one by one, and the two halves behave very differently.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#inference" }, { "name": "Prefill phase", "slug": "prefill-phase", "def": "Processing all prompt tokens in one batched forward pass while computing and caching their keys and values. It is compute-bound, and its length mostly sets the time to first token.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#prefill-phase" }, { "name": "Decode phase", "slug": "decode-phase", "def": "Generating output one token at a time. Each step is a forward pass over a single new token that reads the whole KV cache, which makes it memory-bandwidth-bound.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#decode-phase" }, { "name": "Prompt processing", "slug": "prompt-processing", "def": "Another name for prefill, sometimes counting tokenization and chat-template formatting too. With long prompts it becomes the main cost of a request.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#prompt-processing" }, { "name": "Time to first token (TTFT)", "slug": "time-to-first-token-ttft", "def": "The delay from sending a request to receiving the first output token: queueing plus prefill, whose final position already yields the first token. It is what users feel as responsiveness.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#time-to-first-token-ttft" }, { "name": "Time per output token (TPOT)", "slug": "time-per-output-token-tpot", "def": "The average time between output tokens after the first. Its inverse is the streaming speed of one request in tokens per second.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#time-per-output-token-tpot" }, { "name": "Inter-token latency", "slug": "inter-token-latency", "def": "The gap between two consecutive streamed tokens. TPOT is its average; its spikes, often caused by other requests' prefills sharing the GPU, are what make streaming stutter.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#inter-token-latency" }, { "name": "KV cache", "slug": "kv-cache", "def": "The stored keys and values of every earlier token in every layer, so each decode step only computes K and V for the newest token. It trades memory for compute and often fills most of a serving GPU.", "concepts": [ "gqa", "mla", "kv-sharing" ], "href": "/models/?sort=kv", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#kv-cache" }, { "name": "KV-cache reuse", "slug": "kv-cache-reuse", "def": "Using keys and values computed for one request in another that starts with the same tokens, which skips that part of prefill entirely.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#kv-cache-reuse" }, { "name": "KV-cache allocation", "slug": "kv-cache-allocation", "def": "How an engine hands out GPU memory for caches. Paged allocation, as in vLLM's PagedAttention, splits it into fixed-size blocks so little memory is reserved but left unused.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#kv-cache-allocation" }, { "name": "KV-cache capacity", "slug": "kv-cache-capacity", "def": "How many tokens of cache fit in GPU memory once the weights are loaded. It caps concurrent sequences times their length; compare the per-token cache figures in the gallery.", "href": "/models/?sort=kv", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#kv-cache-capacity" }, { "name": "Prefix caching", "slug": "prefix-caching", "def": "Keeping cache blocks for common prompt beginnings (system prompts, few-shot examples, earlier turns) and matching new requests against them automatically.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#prefix-caching" }, { "name": "Prompt caching", "slug": "prompt-caching", "def": "The API feature built on prefix caching: a repeated prompt prefix is billed at a discount and answered faster. The cached part must usually be identical and at the start of the prompt.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#prompt-caching" }, { "name": "Cache eviction", "slug": "cache-eviction", "def": "Removing cache blocks when memory runs out, usually the least recently used prefixes first. An evicted prefix has to be recomputed if it is needed again.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#cache-eviction" }, { "name": "Cache preemption", "slug": "cache-preemption", "def": "Pausing a running sequence and freeing its cache, either by swapping it to CPU memory or by dropping it and recomputing later, so other requests can proceed when memory is full.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#cache-preemption" }, { "name": "Attention computation", "slug": "attention-computation", "def": "The QKᵀ, softmax and weighted-sum work inside each attention layer. It grows with context length; fused kernels like FlashAttention compute it without ever storing the full score matrix.", "concepts": [ "swa", "deepseek-sparse-attention" ], "url": "https://llm-atlas-7or.pages.dev/learn/inference/#attention-computation" }, { "name": "Compute-bound workloads", "slug": "compute-bound-workloads", "def": "Work whose speed is capped by arithmetic throughput rather than memory. Prefill and large-batch matrix multiplications are compute-bound.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#compute-bound-workloads" }, { "name": "Memory-bandwidth-bound workloads", "slug": "memory-bandwidth-bound-workloads", "def": "Work capped by how fast data can stream from memory. Small-batch decode is the classic case: each token needs every weight and the whole cache to be read for very little maths.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#memory-bandwidth-bound-workloads" }, { "name": "Speculative decoding", "slug": "speculative-decoding", "def": "A fast drafter proposes several tokens, and the large model checks them all in one forward pass, keeping the longest accepted run. The output distribution is unchanged, but each expensive pass can yield several tokens.", "concepts": [ "mtp" ], "url": "https://llm-atlas-7or.pages.dev/learn/inference/#speculative-decoding" }, { "name": "Draft model", "slug": "draft-model", "def": "The cheap proposer in speculative decoding: a smaller model from the same family, extra prediction heads, a multi-token-prediction module, or even n-gram lookups from the prompt.", "concepts": [ "mtp" ], "url": "https://llm-atlas-7or.pages.dev/learn/inference/#draft-model" }, { "name": "Token acceptance", "slug": "token-acceptance", "def": "The verification step of speculative decoding, and the rate at which drafted tokens are accepted. Higher acceptance means more tokens per expensive forward pass.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#token-acceptance" }, { "name": "Model execution overhead", "slug": "model-execution-overhead", "def": "Time spent outside the core maths: scheduler logic, kernel launches, sampling and CPU-GPU synchronisation. At small batch sizes it can eat a large share of each step; CUDA graphs reduce it.", "url": "https://llm-atlas-7or.pages.dev/learn/inference/#model-execution-overhead" } ] }, { "n": 5, "slug": "batching", "name": "Batching & scheduling", "tagline": "Sharing one GPU between many conversations without making anyone wait too long.", "track": "serving", "intro": [ "Reading a model's weights from memory is the expensive part of decoding, and the cost is the same whether those weights are used for one request or sixty-four. Batching serves many requests per read, which is why one GPU can stream to dozens of users at once.", "The hard part is scheduling. Requests arrive at random and finish at different lengths. Continuous batching refills a slot the moment a sequence finishes, and the scheduler's policy decides who runs, who waits and how the tail latency looks." ], "figure": { "title": "Static vs continuous batching", "description": "Eight requests of different lengths share four GPU slots. With static batching the whole batch waits for its longest member, leaving hatched slots idle. Continuous batching starts the next request the moment a slot frees up. Watch the utilisation and finish times." }, "url": "https://llm-atlas-7or.pages.dev/learn/batching/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/batching.md", "terms": [ { "name": "Batching", "slug": "batching", "def": "Processing several requests in the same forward pass. Weights are read from memory once and reused for every request in the batch, the main lever for serving throughput.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#batching" }, { "name": "Sequence batching", "slug": "sequence-batching", "def": "Grouping multiple token sequences into one batch, by padding them to a common length or packing them end to end.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#sequence-batching" }, { "name": "Static batching", "slug": "static-batching", "def": "Collect a fixed batch, run it to completion, then start the next. Simple, but short requests wait for the longest one and finished slots sit empty.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#static-batching" }, { "name": "Dynamic batching", "slug": "dynamic-batching", "def": "Forming batches on the fly from whatever requests arrive within a short time window or up to a size limit, trading a little waiting for better utilisation.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#dynamic-batching" }, { "name": "Continuous batching", "slug": "continuous-batching", "def": "Rebuilding the batch at every decode step: finished sequences leave immediately and waiting requests join. Also called in-flight or iteration-level batching; it is standard in vLLM, SGLang and TensorRT-LLM.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#continuous-batching" }, { "name": "Batch size", "slug": "batch-size", "def": "How many sequences, or tokens, are processed together in one step. Bigger batches raise throughput until compute saturates, but slow each request and need more KV-cache memory.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#batch-size" }, { "name": "Batched forward pass", "slug": "batched-forward-pass", "def": "One forward pass over the whole batch, with tokens from different requests stacked so each weight matrix is multiplied once for all of them.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#batched-forward-pass" }, { "name": "Batch scheduling", "slug": "batch-scheduling", "def": "Deciding at every step which requests run, how many prompt tokens to mix in with decode tokens, and who waits.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#batch-scheduling" }, { "name": "Request admission", "slug": "request-admission", "def": "Deciding whether a newly arrived request can start now, given free cache memory and the batch budget, or must wait in the queue.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#request-admission" }, { "name": "Inference scheduler", "slug": "inference-scheduler", "def": "The engine component that makes batching, admission and preemption decisions every iteration. Its policy shapes throughput, fairness and tail latency.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#inference-scheduler" }, { "name": "Request queue", "slug": "request-queue", "def": "Requests that have arrived but are not yet running. Its depth is an early warning of overload.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#request-queue" }, { "name": "Queueing delay", "slug": "queueing-delay", "def": "Time a request waits before its prefill starts. Under load it can be far larger than the compute time, and it is often the hidden part of TTFT.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#queueing-delay" }, { "name": "Concurrency", "slug": "concurrency", "def": "The number of requests in flight at once, running or waiting. Throughput rises with concurrency until the GPU or its memory saturates.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#concurrency" }, { "name": "Concurrent inference", "slug": "concurrent-inference", "def": "Serving many independent requests on the same model instance at the same time through batching, rather than one after another.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#concurrent-inference" }, { "name": "Active sequences", "slug": "active-sequences", "def": "Sequences in the running batch with their KV cache resident on the GPU. Their number is bounded by memory and the engine's maximum batch setting.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#active-sequences" }, { "name": "Sequence completion", "slug": "sequence-completion", "def": "A sequence reaching a stopping condition. With continuous batching its slot and cache blocks are handed to the next request right away.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#sequence-completion" }, { "name": "Dynamic batch membership", "slug": "dynamic-batch-membership", "def": "The set of sequences in a batch changing from step to step. Continuous batching relies on it, and it requires per-sequence cache bookkeeping.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#dynamic-batch-membership" }, { "name": "Arithmetic intensity", "slug": "arithmetic-intensity", "def": "Floating-point operations done per byte read from memory. Batch-1 decode does about one per byte; batching raises it roughly in proportion to batch size until the hardware becomes compute-bound.", "href": "/learn/gpu/#figure", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#arithmetic-intensity" }, { "name": "Weight reuse", "slug": "weight-reuse", "def": "Using each weight loaded from memory for many tokens. That reuse is exactly what batching buys.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#weight-reuse" }, { "name": "GPU utilization", "slug": "gpu-utilization", "def": "How busy the GPU is. A busy reading in nvidia-smi only means some kernel was running, not that the tensor cores were saturated; model FLOPs utilisation (MFU) is the stricter measure.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#gpu-utilization" }, { "name": "Scheduling policy", "slug": "scheduling-policy", "def": "The rule for ordering work: first come first served, shortest job first, priority classes, or fair shares across tenants.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#scheduling-policy" }, { "name": "Request preemption", "slug": "request-preemption", "def": "Interrupting a running request so a waiting or higher-priority one can run, usually by evicting its cache to be swapped back or recomputed later.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#request-preemption" }, { "name": "Tail latency", "slug": "tail-latency", "def": "The latency of the slowest requests, measured at p95 or p99. Batching choices that improve the average can make the tail worse.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#tail-latency" }, { "name": "Throughput-versus-latency trade-off", "slug": "throughput-versus-latency-trade-off", "def": "Bigger batches serve more tokens per second in total but slow every individual stream. Every deployment picks a point on this curve.", "url": "https://llm-atlas-7or.pages.dev/learn/batching/#throughput-versus-latency-trade-off" } ] }, { "n": 6, "slug": "distributed", "name": "Distributed LLM inference", "tagline": "When one GPU is not enough: splitting, copying and routing models across a cluster.", "track": "serving", "intro": [ "Large models do not fit on one accelerator, and popular ones need far more throughput than one can provide. There are four basic ways to spread the work: copy the model (data parallelism), split each layer (tensor), split the stack of layers (pipeline), or spread the experts (expert parallelism).", "Above the model sits a routing layer. Sending a request to a replica that already holds its prompt prefix in cache, keeping prefill and decode on separate pools, and refusing work before the system tips over all matter as much as raw GPU count." ], "figure": { "title": "Four ways to split a model", "description": "A four-layer model on four GPUs. Each square is one quarter of a layer's weights. Switch modes to watch the pieces move: data parallelism copies everything, tensor parallelism slices every layer, pipeline parallelism stacks layers across GPUs, and expert parallelism spreads the experts." }, "url": "https://llm-atlas-7or.pages.dev/learn/distributed/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/distributed.md", "terms": [ { "name": "Distributed inference", "slug": "distributed-inference", "def": "Running one model across several accelerators, or many copies across a cluster, because it does not fit on one device or one device is too slow.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#distributed-inference" }, { "name": "Model parallelism", "slug": "model-parallelism", "def": "Any scheme that splits a single model across devices: by tensor, by layer or by expert.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#model-parallelism" }, { "name": "Tensor parallelism", "slug": "tensor-parallelism", "def": "Splitting each weight matrix across GPUs so every layer runs on all of them at once, with an all-reduce after attention and the FFN. Low latency, but it needs very fast links such as NVLink, so it usually stays inside one server.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#tensor-parallelism" }, { "name": "Pipeline parallelism", "slug": "pipeline-parallelism", "def": "Putting consecutive groups of layers on different devices and passing activations between them. Communication is light, but stages need a steady flow of micro-batches to avoid idle \"bubbles\".", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#pipeline-parallelism" }, { "name": "Data parallelism", "slug": "data-parallelism", "def": "Running full copies of the model on different devices, each handling different requests or training batches. It scales throughput, not the speed of any one request.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#data-parallelism" }, { "name": "Data-parallel serving", "slug": "data-parallel-serving", "def": "Data parallelism at inference time: several replicas behind a load balancer. Large MoE deployments often run attention data-parallel and experts expert-parallel.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#data-parallel-serving" }, { "name": "Expert parallelism", "slug": "expert-parallelism", "def": "Placing different MoE experts on different GPUs, so each token is shipped with an all-to-all exchange to the GPUs that hold its chosen experts.", "concepts": [ "moe" ], "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#expert-parallelism" }, { "name": "Model sharding", "slug": "model-sharding", "def": "Physically splitting the weights into pieces (shards) stored on different devices, whatever parallelism scheme decides the split.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#model-sharding" }, { "name": "Model replication", "slug": "model-replication", "def": "Keeping several identical copies of a model to serve more traffic or to survive failures.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#model-replication" }, { "name": "Multi-GPU inference", "slug": "multi-gpu-inference", "def": "One model instance spanning several GPUs in a single server, usually with tensor parallelism over NVLink.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#multi-gpu-inference" }, { "name": "Multi-node inference", "slug": "multi-node-inference", "def": "One model instance spanning several servers, needed for the very largest models. It depends on InfiniBand or RoCE networking and usually combines pipeline or expert parallelism across nodes.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#multi-node-inference" }, { "name": "Distributed model serving", "slug": "distributed-model-serving", "def": "The whole system for serving models on a cluster: replicas, routers, autoscaling, caches and health checks.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#distributed-model-serving" }, { "name": "Distributed request routing", "slug": "distributed-request-routing", "def": "Choosing which replica or worker handles each incoming request.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#distributed-request-routing" }, { "name": "Cache-aware routing", "slug": "cache-aware-routing", "def": "Sending a request to the replica that already holds its prefix in KV cache, so it skips recomputing that prefill.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#cache-aware-routing" }, { "name": "Prefix-aware routing", "slug": "prefix-aware-routing", "def": "Cache-aware routing that hashes or matches prompt prefixes, such as the system prompt and conversation so far, to choose a replica.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#prefix-aware-routing" }, { "name": "Cluster scheduling", "slug": "cluster-scheduling", "def": "Allocating GPUs and nodes to models and jobs across a cluster, for example with Kubernetes and GPU-aware schedulers.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#cluster-scheduling" }, { "name": "Global routing", "slug": "global-routing", "def": "Routing across regions or clusters, weighing latency to the user, spare capacity and cost.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#global-routing" }, { "name": "Load balancing", "slug": "load-balancing", "def": "Spreading requests across replicas to avoid hot spots. LLM balancers should look at queue depth and cache usage, not just request counts, because requests vary so much in size.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#load-balancing" }, { "name": "Capacity-aware routing", "slug": "capacity-aware-routing", "def": "Routing that uses live capacity signals such as free cache blocks, queue length or tokens in flight.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#capacity-aware-routing" }, { "name": "Prefill/decode disaggregation", "slug": "prefill-decode-disaggregation", "def": "Running prefill and decode on separate GPU pools, because one is compute-bound and the other bandwidth-bound, and transferring the KV cache between them. It stops long prompts from stalling streams already in progress.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#prefill-decode-disaggregation" }, { "name": "Disaggregated serving", "slug": "disaggregated-serving", "def": "The general design of splitting inference stages (prefill, decode, sometimes attention and experts) onto hardware pools that scale independently.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#disaggregated-serving" }, { "name": "Distributed KV caching", "slug": "distributed-kv-caching", "def": "Storing and sharing KV caches across machines, in CPU memory, SSDs or a dedicated cache tier, so a prefix computed once can be reused cluster-wide.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#distributed-kv-caching" }, { "name": "High-speed interconnects", "slug": "high-speed-interconnects", "def": "Links like NVLink and NVSwitch inside a server and InfiniBand or RoCE between servers. Their bandwidth decides which parallelism schemes are practical.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#high-speed-interconnects" }, { "name": "Multi-tenant inference", "slug": "multi-tenant-inference", "def": "Serving many customers or applications on shared infrastructure, with isolation, quotas and fairness between them.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#multi-tenant-inference" }, { "name": "Inference quality of service (QoS)", "slug": "inference-quality-of-service-qos", "def": "Latency and throughput guarantees or priorities per tenant or request class, such as interactive traffic ahead of batch jobs.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#inference-quality-of-service-qos" }, { "name": "Admission control", "slug": "admission-control", "def": "Rejecting or delaying requests at the edge when the system is saturated, so the requests that are accepted still meet their latency targets.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#admission-control" }, { "name": "Autoscaling", "slug": "autoscaling", "def": "Adding or removing replicas as load changes. Loading large weights takes minutes, so predictive scaling and warm pools matter.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#autoscaling" }, { "name": "High availability", "slug": "high-availability", "def": "Staying up through failures with redundant replicas, health checks, retries and failover across zones.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#high-availability" }, { "name": "Accelerator-aware scheduling", "slug": "accelerator-aware-scheduling", "def": "Placing workloads according to accelerator type, memory and interconnect, for example putting a large model only on nodes with eight NVLinked GPUs.", "url": "https://llm-atlas-7or.pages.dev/learn/distributed/#accelerator-aware-scheduling" } ] }, { "n": 7, "slug": "moe", "name": "Mixture of Experts (MoE)", "tagline": "A huge model that only wakes up a small part of itself for each token.", "track": "serving", "intro": [ "In a Mixture-of-Experts layer, the single feed-forward network is replaced by many smaller expert networks and a router. For every token the router picks a handful of experts, so a model can hold hundreds of billions of parameters while each token only pays for a few tens of billions.", "The catch moves to systems. All experts must sit in memory, tokens have to be shuffled to wherever their experts live, and an unbalanced router leaves some GPUs overloaded and others idle. Most large open-weight models in the gallery are now MoEs." ], "figure": { "title": "A router at work", "description": "Tokens arrive one by one. The router scores all eight experts and sends each token to its top two. Turn load balancing off to watch a few experts get swamped while others sit idle; the load bars underneath show the imbalance." }, "url": "https://llm-atlas-7or.pages.dev/learn/moe/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/moe.md", "terms": [ { "name": "Mixture of Experts (MoE)", "slug": "mixture-of-experts-moe", "def": "An architecture where the FFN in each block is replaced by many expert FFNs plus a router that sends each token to only a few of them. Total parameters grow without a matching rise in compute per token.", "concepts": [ "moe", "latent-moe" ], "href": "/models/?type=moe", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#mixture-of-experts-moe" }, { "name": "Dense models", "slug": "dense-models", "def": "Models where every parameter is used for every token. Simpler to train and serve, but compute per token rises with total size.", "href": "/models/?type=dense", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#dense-models" }, { "name": "Sparse MoE models", "slug": "sparse-moe-models", "def": "MoE models where only a small subset of experts runs per token, for example 8 of 256. Most of the largest open-weight models are now sparse MoEs.", "href": "/models/?type=moe", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#sparse-moe-models" }, { "name": "Experts", "slug": "experts", "def": "The individual FFNs inside an MoE layer. Nobody assigns them topics; any specialisation emerges in training and tends to follow token types and syntax more than subjects.", "concepts": [ "moe" ], "url": "https://llm-atlas-7or.pages.dev/learn/moe/#experts" }, { "name": "Expert MLPs", "slug": "expert-mlps", "def": "The MLP each expert is, typically a SwiGLU FFN much narrower than the dense FFN it replaces. Many designs also add one or more shared experts that every token uses.", "concepts": [ "latent-moe" ], "url": "https://llm-atlas-7or.pages.dev/learn/moe/#expert-mlps" }, { "name": "MoE routing", "slug": "moe-routing", "def": "Deciding which experts process each token at each MoE layer.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#moe-routing" }, { "name": "Router network", "slug": "router-network", "def": "A small linear layer that maps each token's hidden state to one score per expert.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#router-network" }, { "name": "Routing logits", "slug": "routing-logits", "def": "The raw per-expert scores from the router, before softmax or sigmoid. Top-k is taken over them.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#routing-logits" }, { "name": "Gating network", "slug": "gating-network", "def": "Another name for the router. \"Gate\" stresses that it also sets how much each chosen expert's output counts.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#gating-network" }, { "name": "Gating weights", "slug": "gating-weights", "def": "The normalised router scores of the selected experts, used to weight each expert's output when combining them.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#gating-weights" }, { "name": "Top-k expert selection", "slug": "top-k-expert-selection", "def": "Choosing the k experts with the highest scores for each token. k is small, often between 1 and 8, however many experts the layer has.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#top-k-expert-selection" }, { "name": "Top-2 routing", "slug": "top-2-routing", "def": "Top-k with k = 2, used by Mixtral and many early MoEs. Each token's output is the weighted sum of two experts.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#top-2-routing" }, { "name": "Active parameters", "slug": "active-parameters", "def": "The parameters actually used for one token: attention, shared parts and the chosen experts. They drive compute per token and are the \"A22B\" in names like 235B-A22B.", "href": "/models/?sort=sparsest", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#active-parameters" }, { "name": "Total parameters", "slug": "total-parameters", "def": "All parameters, every expert included. They drive how much memory is needed to hold the model, even though most sit idle for any one token.", "href": "/models/?sort=largest", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#total-parameters" }, { "name": "Expert dispatch", "slug": "expert-dispatch", "def": "Sending tokens to their chosen experts: regrouping them by expert and, under expert parallelism, moving them to other GPUs.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#expert-dispatch" }, { "name": "Expert combine", "slug": "expert-combine", "def": "Gathering expert outputs back into the original token order and summing them with the gating weights.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#expert-combine" }, { "name": "Token permutation", "slug": "token-permutation", "def": "Reordering a batch so that tokens for the same expert sit next to each other, which allows efficient grouped matrix multiplications; it is reversed after the combine.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#token-permutation" }, { "name": "Grouped matrix multiplication", "slug": "grouped-matrix-multiplication", "def": "One kernel that runs many independent matrix multiplications of different sizes, one per expert, instead of launching each separately. Essential for fast MoE layers.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#grouped-matrix-multiplication" }, { "name": "Expert load balancing", "slug": "expert-load-balancing", "def": "Keeping experts evenly used, through auxiliary balancing losses, per-expert router biases (as in DeepSeek's auxiliary-loss-free method) or capacity limits.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#expert-load-balancing" }, { "name": "Expert placement", "slug": "expert-placement", "def": "Deciding which experts live on which GPUs. Popular experts can be replicated to spread their load.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#expert-placement" }, { "name": "Inter-GPU communication", "slug": "inter-gpu-communication", "def": "Data exchanged between GPUs during a forward pass: all-reduce for tensor parallelism, all-to-all for expert parallelism. It often caps MoE throughput.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#inter-gpu-communication" }, { "name": "All-to-all communication", "slug": "all-to-all-communication", "def": "A collective operation where every GPU sends a different piece of data to every other GPU. Used to dispatch tokens to experts and bring the results back.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#all-to-all-communication" }, { "name": "Routing imbalance", "slug": "routing-imbalance", "def": "When the router sends far more tokens to some experts than others, creating stragglers, dropped tokens or idle GPUs.", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#routing-imbalance" }, { "name": "Sparse computation", "slug": "sparse-computation", "def": "Doing only a fraction of the possible work for each input. MoE does it for the FFN; sparse attention applies the same idea to attention.", "concepts": [ "deepseek-sparse-attention", "qwen-sparse-attention" ], "url": "https://llm-atlas-7or.pages.dev/learn/moe/#sparse-computation" }, { "name": "Expert parallelism", "slug": "expert-parallelism", "def": "Spreading experts across GPUs: each holds a subset and receives the tokens routed to it through all-to-all. Large deployments pair it with data-parallel attention.", "href": "/learn/distributed/#expert-parallelism", "url": "https://llm-atlas-7or.pages.dev/learn/moe/#expert-parallelism" } ] }, { "n": 8, "slug": "gpu", "name": "GPU & CUDA engineering", "tagline": "The hardware realities that decide how fast a model can run.", "track": "serving", "intro": [ "A GPU can do roughly a thousand trillion half-precision operations per second, but it can only read a few trillion bytes per second from its main memory. Whether a workload is limited by the maths or by the memory depends on how many operations it does per byte it reads.", "That ratio, arithmetic intensity, explains why batch-1 decode leaves most of the chip idle and why batching, fusion and quantization help so much. Kernel engineering is largely the craft of moving fewer bytes and keeping data close to the compute." ], "figure": { "title": "The roofline", "description": "Attainable speed for an H100-class GPU (about 990 dense BF16 TFLOPS, 3.35 TB/s of memory bandwidth; rounded figures). Drag the batch size and watch decode climb the sloped, memory-bound part of the roof until it reaches the flat compute ceiling. Below: the memory hierarchy, from tiny and fast to huge and slow." }, "url": "https://llm-atlas-7or.pages.dev/learn/gpu/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/gpu.md", "terms": [ { "name": "GPU computing", "slug": "gpu-computing", "def": "Using graphics processors for general parallel computing. Thousands of simple cores plus specialised tensor cores make them ideal for neural-network matrix maths.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#gpu-computing" }, { "name": "NVIDIA GPU", "slug": "nvidia-gpu", "def": "The dominant accelerator family for AI (A100, H100, H200, B200 and successors), combining tensor cores, high-bandwidth memory and NVLink with the CUDA software ecosystem.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#nvidia-gpu" }, { "name": "CUDA", "slug": "cuda", "def": "NVIDIA's platform and programming model for its GPUs: compiler, runtime, driver API and libraries such as cuBLAS and cuDNN. Most AI frameworks run on top of it.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#cuda" }, { "name": "CUDA C++", "slug": "cuda-cpp", "def": "The C++ dialect for writing CUDA kernels, with extensions for launching grids of threads and managing GPU memory.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#cuda-cpp" }, { "name": "Triton language", "slug": "triton-language", "def": "OpenAI's Python-based language and compiler for writing GPU kernels at the level of tiles. It handles much of the low-level scheduling, so custom fused kernels take far less code than CUDA C++.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#triton-language" }, { "name": "GPU kernels", "slug": "gpu-kernels", "def": "Functions that run on the GPU across many threads in parallel, such as a matrix multiplication, a softmax or a fused attention kernel.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#gpu-kernels" }, { "name": "Kernel launches", "slug": "kernel-launches", "def": "The CPU asking the GPU to start a kernel. Each launch costs a few microseconds, which adds up over thousands of small kernels per token; CUDA graphs record and replay them to cut the cost.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#kernel-launches" }, { "name": "Kernel fusion", "slug": "kernel-fusion", "def": "Combining several operations into one kernel so intermediate results stay in registers or shared memory instead of round-tripping through main memory. FlashAttention is the best-known example.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#kernel-fusion" }, { "name": "Fused sampling", "slug": "fused-sampling", "def": "Doing temperature, top-k or top-p filtering and the random draw in one GPU kernel, avoiding a full-vocabulary sort and extra trips to the CPU.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#fused-sampling" }, { "name": "GPU memory", "slug": "gpu-memory", "def": "Memory attached to the GPU. It must hold the weights, KV cache, activations and workspace, and running out of it is the most common serving limit.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#gpu-memory" }, { "name": "VRAM", "slug": "vram", "def": "The common name for GPU memory capacity, such as 80 GB on an H100 SXM or 141 GB on an H200.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#vram" }, { "name": "GPU memory hierarchy", "slug": "gpu-memory-hierarchy", "def": "The levels from registers (fastest, tiny) through shared memory and L1 per streaming multiprocessor, the shared L2 cache and HBM, out to host RAM over PCIe. Fast kernels keep data as high in the hierarchy as they can.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#gpu-memory-hierarchy" }, { "name": "Memory bandwidth", "slug": "memory-bandwidth", "def": "How many bytes per second can move between HBM and the compute units, about 3.35 TB/s on an H100 SXM. It bounds decode speed: tokens per second is roughly bandwidth divided by the bytes read per token.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#memory-bandwidth" }, { "name": "Memory traffic", "slug": "memory-traffic", "def": "The total bytes a kernel reads and writes. Cutting it through fusion, quantization or smaller KV caches is often the most effective optimisation.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#memory-traffic" }, { "name": "Memory locality", "slug": "memory-locality", "def": "Accessing data that is nearby or recently used, so caches hit and loads coalesce. Poor locality wastes bandwidth.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#memory-locality" }, { "name": "GPU compute utilization", "slug": "gpu-compute-utilization", "def": "The share of peak FLOPs actually achieved. Model FLOPs utilisation of 40 to 60 percent is good for training; batch-1 decode reaches far less.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#gpu-compute-utilization" }, { "name": "Memory utilization", "slug": "memory-utilization", "def": "Either the share of memory capacity in use or the share of peak bandwidth achieved. Serving engines deliberately reserve most free memory for KV cache, so high capacity use is normal.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#memory-utilization" }, { "name": "CPU–GPU data transfers", "slug": "cpu-gpu-data-transfers", "def": "Copies between host and device over PCIe or NVLink-C2C. They are slow compared with HBM, so engines keep them out of the hot loop; cache offloading uses them on purpose.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#cpu-gpu-data-transfers" }, { "name": "Tensor operations", "slug": "tensor-operations", "def": "Operations on multi-dimensional arrays: matrix multiplications, elementwise maths, reductions and reshapes. Frameworks build models from them and compilers fuse them.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#tensor-operations" }, { "name": "Matrix-vector multiplication", "slug": "matrix-vector-multiplication", "def": "Multiplying a matrix by a single vector (GEMV). Batch-1 decode is effectively a chain of these, which is why it is bandwidth-bound.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#matrix-vector-multiplication" }, { "name": "Matrix-matrix multiplication", "slug": "matrix-matrix-multiplication", "def": "Multiplying two matrices (GEMM). Batching and prefill turn GEMVs into GEMMs, which keep tensor cores busy.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#matrix-matrix-multiplication" }, { "name": "Parallel execution", "slug": "parallel-execution", "def": "Running work simultaneously at every level: threads in a warp, warps on a multiprocessor, multiprocessors on a GPU, GPUs in a cluster, and communication overlapped with compute.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#parallel-execution" }, { "name": "GPU profiling", "slug": "gpu-profiling", "def": "Measuring where GPU time goes with tools such as Nsight Systems and the PyTorch profiler, to find stalls, idle gaps and slow kernels.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#gpu-profiling" }, { "name": "Kernel profiling", "slug": "kernel-profiling", "def": "Inspecting a single kernel in depth with Nsight Compute: achieved bandwidth, occupancy, tensor-core use and the reasons it stalls.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#kernel-profiling" }, { "name": "Accelerator clusters", "slug": "accelerator-clusters", "def": "Many accelerators networked together, thousands of GPUs or TPUs, for training and large-scale inference.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#accelerator-clusters" }, { "name": "NVIDIA H100", "slug": "nvidia-h100", "def": "NVIDIA's Hopper-generation data-centre GPU from 2022: 80 GB of HBM3, about 3.35 TB/s of bandwidth, roughly 1 PFLOP of dense BF16 and FP8 support. The reference chip for most LLM work of 2023 to 2025.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#nvidia-h100" }, { "name": "AWS Trainium", "slug": "aws-trainium", "def": "Amazon's own AI accelerator chips for training and inference on AWS, programmed through the Neuron SDK.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#aws-trainium" }, { "name": "Google TPU", "slug": "google-tpu", "def": "Google's Tensor Processing Units: systolic-array accelerators linked into large pods and programmed through JAX and XLA. Gemini and Gemma are trained on them.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#google-tpu" }, { "name": "Hardware-specific optimization", "slug": "hardware-specific-optimization", "def": "Tuning kernels, number formats and parallelism for a particular chip, such as FP8 on Hopper, FP4 on Blackwell, or tile sizes matched to a multiprocessor's shared memory.", "url": "https://llm-atlas-7or.pages.dev/learn/gpu/#hardware-specific-optimization" } ] }, { "n": 9, "slug": "agents", "name": "Agent architecture & internals", "tagline": "A model in a loop, with tools, state and a reason to stop.", "track": "agents", "intro": [ "An agent is not a special kind of model. It is ordinary software that calls a model in a loop: build the context, ask the model what to do, run the tool it asks for, add the result to the context, and repeat until the model gives an answer or a limit is hit.", "Everything interesting lives in the harness around that loop: which instructions and tools the model sees, how state is stored, when to hand work to a sub-agent, and how to stop. The same model can look brilliant or useless depending on that scaffolding." ], "figure": { "title": "The agent loop", "description": "An agent booking a meeting. Each lap the runtime assembles context, the model decides, a tool runs, and its result goes back into context. On the last lap the model answers instead of calling a tool and the loop ends. Step through it or let it play." }, "url": "https://llm-atlas-7or.pages.dev/learn/agents/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/agents.md", "terms": [ { "name": "AI agents", "slug": "ai-agents", "def": "Systems where an LLM decides which actions to take, calls tools, looks at the results and repeats until a goal is reached, instead of answering once.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#ai-agents" }, { "name": "Agentic systems", "slug": "agentic-systems", "def": "Software in which LLM decisions steer part of the control flow, from a single tool-calling loop to large multi-agent workflows.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agentic-systems" }, { "name": "Agent systems engineering", "slug": "agent-systems-engineering", "def": "The discipline of making agents dependable in production: runtimes, state, tools, evaluation, security and cost control.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-systems-engineering" }, { "name": "Agent architecture", "slug": "agent-architecture", "def": "How an agent is put together: its model, instructions, tools, memory, control loop and any sub-agents or workflow graph around it.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-architecture" }, { "name": "Agent runtime", "slug": "agent-runtime", "def": "The software that executes agents: it runs the loop, calls the model, dispatches tools, saves state and enforces limits.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-runtime" }, { "name": "Agent harness", "slug": "agent-harness", "def": "The scaffolding that turns a model into an agent: prompt assembly, tool definitions, output parsing, the loop and its stop rules. The same model can perform very differently in different harnesses.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-harness" }, { "name": "Agent control loop", "slug": "agent-control-loop", "def": "The central cycle: assemble context, call the model, run any requested tools and append their results, then repeat until the model answers or a limit is reached.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-control-loop" }, { "name": "Reasoning loop", "slug": "reasoning-loop", "def": "The model's think, act, observe pattern inside the control loop, popularised as ReAct, where intermediate reasoning guides the next action.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#reasoning-loop" }, { "name": "Agent orchestration", "slug": "agent-orchestration", "def": "Coordinating agents, tools and steps: what runs when, in which order or in parallel, and how results flow between them.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-orchestration" }, { "name": "Tool-calling loop", "slug": "tool-calling-loop", "def": "The control loop built on function calling: the model emits tool calls, the runtime runs them and returns the results as messages.", "href": "/learn/tool-calling/", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#tool-calling-loop" }, { "name": "Agent state machines", "slug": "agent-state-machines", "def": "Modelling an agent as explicit states and transitions, such as planning, acting, awaiting approval and done, which makes behaviour predictable and recovery easier.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-state-machines" }, { "name": "Agent lifecycle", "slug": "agent-lifecycle", "def": "The stages of one agent instance: created, configured, running, paused or waiting for input, completed or failed, and cleaned up.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-lifecycle" }, { "name": "Agent sessions", "slug": "agent-sessions", "def": "A bounded run or conversation with its own message history, state and resources.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-sessions" }, { "name": "Agent identity", "slug": "agent-identity", "def": "Who the agent is to other systems: its credentials, permissions and audit identity, kept separate from the user it acts for.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-identity" }, { "name": "Agent configuration", "slug": "agent-configuration", "def": "The settings that define an agent: model, sampling settings, instructions, enabled tools, limits and memory options.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-configuration" }, { "name": "Agent instructions", "slug": "agent-instructions", "def": "The system prompt and guidelines that set the agent's role, goals, constraints and rules for using tools.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-instructions" }, { "name": "Agent capabilities", "slug": "agent-capabilities", "def": "What an agent can actually do. It is set by the tools and permissions it is given, not by the model alone.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-capabilities" }, { "name": "Agent state", "slug": "agent-state", "def": "Everything needed to continue an agent from where it is: messages, intermediate results, its plan, counters and pending tool calls.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-state" }, { "name": "Tool selection", "slug": "tool-selection", "def": "The model choosing which tool to call, or whether to call one at all, from the tool descriptions and the current context.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#tool-selection" }, { "name": "Tool invocation", "slug": "tool-invocation", "def": "Actually calling the chosen tool with the arguments the model produced.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#tool-invocation" }, { "name": "Tool result handling", "slug": "tool-result-handling", "def": "Formatting a tool's output, or its error, back into the context, often trimming or summarising large results first.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#tool-result-handling" }, { "name": "Tool dispatch", "slug": "tool-dispatch", "def": "Routing a tool call to its implementation, whether a local function, an MCP server, an HTTP API or a sandboxed process, possibly several in parallel.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#tool-dispatch" }, { "name": "Agent termination", "slug": "agent-termination", "def": "Ending the loop: the model gives a final answer, a step, cost or time limit is hit, an error cannot be recovered, or a person stops it.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-termination" }, { "name": "Subagents", "slug": "subagents", "def": "Agents started by another agent for a sub-task. Each gets its own context window and returns only a summary, which keeps the parent's context small.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#subagents" }, { "name": "Agent delegation", "slug": "agent-delegation", "def": "Handing a task to another agent, including deciding what context to pass along and what result to expect back.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-delegation" }, { "name": "Multi-agent systems", "slug": "multi-agent-systems", "def": "Systems of several agents with distinct roles that cooperate, such as a planner with workers or a writer with a reviewer.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#multi-agent-systems" }, { "name": "Multi-agent collaboration", "slug": "multi-agent-collaboration", "def": "Agents working toward a shared goal by dividing the work, sharing results and checking each other.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#multi-agent-collaboration" }, { "name": "Agent coordination", "slug": "agent-coordination", "def": "The mechanisms that keep several agents consistent: shared state, locks, turn-taking or a supervising agent.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-coordination" }, { "name": "Agent-to-agent communication", "slug": "agent-to-agent-communication", "def": "Messages exchanged between agents, directly or through shared memory, increasingly over standard protocols such as A2A.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#agent-to-agent-communication" }, { "name": "Task decomposition", "slug": "task-decomposition", "def": "Breaking a goal into smaller steps or sub-tasks that can be run, delegated or parallelised.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#task-decomposition" }, { "name": "Autonomous task execution", "slug": "autonomous-task-execution", "def": "An agent carrying a task through many steps without human input, inside guardrails and limits.", "url": "https://llm-atlas-7or.pages.dev/learn/agents/#autonomous-task-execution" } ] }, { "n": 10, "slug": "tool-calling", "name": "Tool calling & protocols", "tagline": "How a model asks for something to be done, and how the answer gets back.", "track": "agents", "intro": [ "A model cannot run code or open a web page. What it can do is emit a structured request: the name of a tool and some JSON arguments. The application validates that request, runs the tool, and feeds the result back as a new message so the model can continue.", "Under the hood that request is ordinary tokens, wrapped in special tokens the model learned in training and laid out by a chat template. On top sit protocols such as MCP, for connecting tools, and AG-UI, for streaming agent activity to a user interface." ], "figure": { "title": "Anatomy of a tool call", "description": "One full round trip between the user, the application, the model and a tool. Step through to see the actual messages at each stage: the tool definition the model is given, the call it emits, the tool result, and the final answer." }, "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/tool-calling.md", "terms": [ { "name": "Function calling", "slug": "function-calling", "def": "A model's ability to output a structured request to run a named function with arguments, instead of plain text.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#function-calling" }, { "name": "Tool definitions", "slug": "tool-definitions", "def": "Declarations of the tools the model may use, passed with the request: name, description and parameter schema.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-definitions" }, { "name": "Tool schemas", "slug": "tool-schemas", "def": "The JSON Schema for a tool's parameters: types, required fields, allowed values and field descriptions. Good schemas prevent bad calls.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-schemas" }, { "name": "Tool descriptions", "slug": "tool-descriptions", "def": "Plain-language text saying what a tool does and when to use it. The model chooses tools mostly from these.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-descriptions" }, { "name": "Function arguments", "slug": "function-arguments", "def": "The values the model fills in for a tool's parameters, emitted as JSON.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#function-arguments" }, { "name": "Tool-call tokens", "slug": "tool-call-tokens", "def": "Special tokens a model is trained to put around tool calls, which the engine uses to detect and parse them.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-call-tokens" }, { "name": "Special tokens", "slug": "special-tokens", "def": "Reserved vocabulary entries with structural meaning, such as begin and end of text, role markers and tool-call markers. Ordinary text never produces them.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#special-tokens" }, { "name": "Control tokens", "slug": "control-tokens", "def": "Special tokens that steer format or behaviour: role switches, turning a thinking mode on or off, end of turn.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#control-tokens" }, { "name": "Message boundary tokens", "slug": "message-boundary-tokens", "def": "Tokens marking where each message starts and ends, such as <|im_start|> and <|im_end|> in ChatML-style templates.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#message-boundary-tokens" }, { "name": "Tool routing tokens", "slug": "tool-routing-tokens", "def": "Tokens or markers that say which tool or output channel the following content belongs to, so the parser can route it.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-routing-tokens" }, { "name": "Chat templates", "slug": "chat-templates", "def": "The model-specific recipe, often a Jinja template shipped with the tokenizer, that turns a list of messages and tools into the exact token sequence the model was trained on. A mismatched template quietly degrades quality.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#chat-templates" }, { "name": "Structured model outputs", "slug": "structured-model-outputs", "def": "Responses guaranteed to match a schema, through constrained decoding or through validation and retry.", "href": "/learn/decoding/#structured-generation", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#structured-model-outputs" }, { "name": "JSON Schema", "slug": "json-schema", "def": "A standard vocabulary for describing the shape of JSON data. It is the common language for tool parameters and structured outputs.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#json-schema" }, { "name": "JSON parsing", "slug": "json-parsing", "def": "Turning the model's emitted text into data. Robust harnesses cope with partial streams, trailing text and malformed JSON.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#json-parsing" }, { "name": "Argument validation", "slug": "argument-validation", "def": "Checking parsed arguments against the schema and business rules before running the tool, and returning a useful error to the model when they fail.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#argument-validation" }, { "name": "Tool authorization", "slug": "tool-authorization", "def": "Deciding whether this agent, acting for this user, may call this tool with these arguments right now.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-authorization" }, { "name": "Tool permissions", "slug": "tool-permissions", "def": "The policy of which tools and scopes an agent has been granted, ideally the least it needs.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-permissions" }, { "name": "Tool execution", "slug": "tool-execution", "def": "Running the tool's implementation, ideally in a sandbox with timeouts and resource limits.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-execution" }, { "name": "Tool-call errors", "slug": "tool-call-errors", "def": "Failures from bad arguments, missing tools, timeouts or exceptions. Reporting them clearly lets the model correct itself on the next step.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-call-errors" }, { "name": "Hallucinated tools", "slug": "hallucinated-tools", "def": "The model calling a tool that does not exist or inventing parameters. Validation catches it; clear definitions and constrained decoding make it rarer.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#hallucinated-tools" }, { "name": "OpenAI-compatible API", "slug": "openai-compatible-api", "def": "The de facto HTTP interface, /v1/chat/completions with messages, tools and tool_calls, that most inference servers and gateways implement so clients can switch providers.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#openai-compatible-api" }, { "name": "AG-UI protocol", "slug": "ag-ui-protocol", "def": "The Agent-User Interaction protocol: an open, event-based standard started by CopilotKit for streaming agent events, such as messages, tool calls and state changes, to front-end apps.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#ag-ui-protocol" }, { "name": "Model Context Protocol (MCP)", "slug": "model-context-protocol-mcp", "def": "An open protocol, introduced by Anthropic, for connecting models to tools and data. MCP servers expose tools, resources and prompts over a standard JSON-RPC interface that any client can use.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#model-context-protocol-mcp" }, { "name": "Agent-to-application communication", "slug": "agent-to-application-communication", "def": "How an agent talks to the app around it: streamed tokens, tool-call events, state updates and requests for user input.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#agent-to-application-communication" }, { "name": "Agent-to-tool communication", "slug": "agent-to-tool-communication", "def": "How an agent reaches its tools: direct function calls, HTTP APIs or protocols such as MCP.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#agent-to-tool-communication" }, { "name": "Tool-call finish reasons", "slug": "tool-call-finish-reasons", "def": "The field in a response that says why generation stopped, such as \"tool_calls\" (run the tools and call me back), \"stop\" or \"length\".", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-call-finish-reasons" }, { "name": "Forced tool selection", "slug": "forced-tool-selection", "def": "Requiring the model to call a particular tool, or any tool, on this turn through the tool_choice setting.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#forced-tool-selection" }, { "name": "Tool choice", "slug": "tool-choice", "def": "The API parameter that controls tool use: auto (the model decides), none, required, or one specific named tool.", "url": "https://llm-atlas-7or.pages.dev/learn/tool-calling/#tool-choice" } ] }, { "n": 11, "slug": "context-memory", "name": "Context, memory & persistence", "tagline": "What the model can see right now, and what the system remembers for later.", "track": "agents", "intro": [ "A model has no memory between calls. Everything it knows about the task has to be in the context window on every request: instructions, tool definitions, the conversation so far, tool results and anything retrieved from storage. The window is large but finite, and filling it costs money and latency.", "Context management is the art of choosing what goes in. Persistence is the other half: saving the agent's state durably, so a long task can survive a crash, be replayed for debugging or pick up again next week." ], "figure": { "title": "Filling a context window", "description": "A 128K-token window split between instructions, tools, retrieved knowledge, conversation history and room for the reply. Add turns and the history grows until it no longer fits. Choose a strategy to see what happens: truncation drops the oldest turns, compaction folds them into a short summary." }, "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/context-memory.md", "terms": [ { "name": "Context window", "slug": "context-window", "def": "The maximum number of tokens a model can attend to at once, prompt and output together, such as 128K or 1M tokens.", "href": "/models/?sort=context", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#context-window" }, { "name": "Context length", "slug": "context-length", "def": "The number of tokens actually in a given request. Cost and latency grow with it, and quality can slip well before the window is full.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#context-length" }, { "name": "Context management", "slug": "context-management", "def": "Deciding what goes into the limited context on each model call: which instructions, history, tool results and retrieved knowledge.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#context-management" }, { "name": "Context assembly", "slug": "context-assembly", "def": "Building the final prompt from its parts in a stable order: instructions, tool definitions, memory, retrieved documents and recent messages. A stable order also keeps prefix caching effective.", "href": "/learn/inference/#prefix-caching", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#context-assembly" }, { "name": "Short-term memory", "slug": "short-term-memory", "def": "Information kept within the current context or session, mainly the recent conversation and working notes.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#short-term-memory" }, { "name": "Long-term memory", "slug": "long-term-memory", "def": "Information kept across sessions in external storage, such as files, databases or vector stores, and brought back when relevant.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#long-term-memory" }, { "name": "Persistent memory", "slug": "persistent-memory", "def": "Memory that survives restarts and sessions, such as user preferences or project facts written to durable storage.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#persistent-memory" }, { "name": "Conversation history", "slug": "conversation-history", "def": "The sequence of past user, assistant and tool messages in a session.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#conversation-history" }, { "name": "Message history", "slug": "message-history", "def": "The raw list of messages sent to the model on each turn. Because the model is stateless, the whole history is sent again on every call, ideally with caching.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#message-history" }, { "name": "Memory retrieval", "slug": "memory-retrieval", "def": "Selecting the stored memories relevant to the current step, by recency, keywords or embedding similarity.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#memory-retrieval" }, { "name": "Retrieval systems", "slug": "retrieval-systems", "def": "Search infrastructure, such as BM25, vector search, hybrid search and rerankers, that finds documents to place into context, as in retrieval-augmented generation (RAG).", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#retrieval-systems" }, { "name": "Context compaction", "slug": "context-compaction", "def": "Shrinking the context while keeping what matters: summarising old turns, dropping stale tool output, holding on to key decisions.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#context-compaction" }, { "name": "Context summarization", "slug": "context-summarization", "def": "Replacing a span of history with a model-written summary. It saves tokens but can lose details.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#context-summarization" }, { "name": "Context truncation", "slug": "context-truncation", "def": "Dropping the oldest or least important content when the context exceeds its budget. Simple, but information disappears abruptly.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#context-truncation" }, { "name": "Token budgets", "slug": "token-budgets", "def": "Allocations of the window to each part (instructions, tools, history, retrieval, output), enforced when the context is assembled.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#token-budgets" }, { "name": "Persistent agent state", "slug": "persistent-agent-state", "def": "Agent state saved to durable storage so a run can pause, survive crashes and resume.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#persistent-agent-state" }, { "name": "Execution history", "slug": "execution-history", "def": "A record of every step an agent took: model calls, tool calls, results and decisions.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#execution-history" }, { "name": "Checkpointing", "slug": "checkpointing", "def": "Saving the agent's state at defined points so it can resume from there instead of starting over.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#checkpointing" }, { "name": "State snapshots", "slug": "state-snapshots", "def": "Point-in-time copies of the full state, useful for debugging, branching and rolling back.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#state-snapshots" }, { "name": "State recovery", "slug": "state-recovery", "def": "Restoring an agent from its last checkpoint after a failure.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#state-recovery" }, { "name": "Execution replay", "slug": "execution-replay", "def": "Re-running a recorded execution, reusing the recorded model and tool results, to debug it or to recover.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#execution-replay" }, { "name": "Deterministic replay", "slug": "deterministic-replay", "def": "Replay that follows the same path every time because nondeterministic results, such as model outputs, clock reads and tool responses, are read from the log instead of recomputed. Workflow engines like Temporal depend on it.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#deterministic-replay" }, { "name": "Nondeterministic execution", "slug": "nondeterministic-execution", "def": "Steps that can give a different result each time: sampled model output, network calls, the current time. They must be recorded to make replay safe.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#nondeterministic-execution" }, { "name": "Durable state", "slug": "durable-state", "def": "State stored where it survives process and machine failures, such as a database or an event log.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#durable-state" }, { "name": "Persistent workflows", "slug": "persistent-workflows", "def": "Long-running workflows whose progress is recorded durably, so they can run for hours or days and survive restarts.", "url": "https://llm-atlas-7or.pages.dev/learn/context-memory/#persistent-workflows" } ] }, { "n": 12, "slug": "backend", "name": "Backend & distributed systems", "tagline": "The plumbing that keeps thousands of agent runs moving and correct.", "track": "production", "intro": [ "An agent platform is a distributed system first and an AI system second. Requests come in through a gateway, a control plane decides what runs where, workers in an execution plane carry out agent loops and tools, and an inference plane turns prompts into tokens.", "Between them sit queues, leases and event streams, and the classic problems come along: work that runs twice, work that never finishes, overloaded dependencies. Idempotency, backpressure and durable state are what keep it honest." ], "figure": { "title": "Three planes of an agent platform", "description": "Requests flow from the gateway through the control plane into a queue, are picked up by workers, and call out to the inference plane for tokens. Raise the arrival rate past what the workers can handle and switch on backpressure to see the difference between a queue that grows forever and one that pushes back." }, "url": "https://llm-atlas-7or.pages.dev/learn/backend/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/backend.md", "terms": [ { "name": "Backend engineering", "slug": "backend-engineering", "def": "Building the server side of an AI product: APIs, data, queues, workers and integrations with model providers.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#backend-engineering" }, { "name": "Control plane", "slug": "control-plane", "def": "The part of a system that decides and manages (configuration, scheduling, placement, policy) rather than doing the work itself.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#control-plane" }, { "name": "Execution plane", "slug": "execution-plane", "def": "Where agent work actually runs: the workers that execute agent loops and tools.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#execution-plane" }, { "name": "Inference plane", "slug": "inference-plane", "def": "The model-serving layer: GPU clusters, inference engines and routers that turn prompts into tokens.", "href": "/learn/distributed/", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#inference-plane" }, { "name": "API gateway", "slug": "api-gateway", "def": "The front door for client requests, handling authentication, rate limits, routing and request shaping.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#api-gateway" }, { "name": "Model gateway", "slug": "model-gateway", "def": "A proxy in front of one or more model providers that unifies their APIs and handles keys, fallbacks, routing, caching and cost tracking.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#model-gateway" }, { "name": "Service boundaries", "slug": "service-boundaries", "def": "The lines between services: what each one owns, its API and its data.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#service-boundaries" }, { "name": "Modular monolith", "slug": "modular-monolith", "def": "One deployable application split into strongly separated modules. It gets clear boundaries without the operational cost of microservices.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#modular-monolith" }, { "name": "Microservices", "slug": "microservices", "def": "Many small, independently deployed services talking over the network. Independent scaling and teams, at the cost of distributed-systems complexity.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#microservices" }, { "name": "Distributed systems", "slug": "distributed-systems", "def": "Systems of several machines cooperating over a network, which must cope with partial failure, latency and the lack of a shared clock.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#distributed-systems" }, { "name": "Distributed orchestration", "slug": "distributed-orchestration", "def": "Coordinating multi-step work across machines, with retries, timeouts and durable state.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#distributed-orchestration" }, { "name": "Worker processes", "slug": "worker-processes", "def": "Processes that pull work from a queue and execute it, scaled separately from the API.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#worker-processes" }, { "name": "Worker pools", "slug": "worker-pools", "def": "Groups of workers sized to demand, sometimes specialised by resource such as CPU, GPU or sandbox slots.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#worker-pools" }, { "name": "Task scheduling", "slug": "task-scheduling", "def": "Deciding when and where tasks run, including delays, priorities and recurring jobs.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#task-scheduling" }, { "name": "Task queues", "slug": "task-queues", "def": "Queues of units of work waiting for workers, which decouple accepting a request from executing it.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#task-queues" }, { "name": "Message queues", "slug": "message-queues", "def": "Infrastructure that holds messages until consumers process them, smoothing bursts between producers and consumers.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#message-queues" }, { "name": "Message brokers", "slug": "message-brokers", "def": "Servers that route and store messages between producers and consumers, such as NATS, RabbitMQ or Kafka.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#message-brokers" }, { "name": "Asynchronous messaging", "slug": "asynchronous-messaging", "def": "Sending a message without waiting for the receiver to process it, so services stay decoupled in time.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#asynchronous-messaging" }, { "name": "Event-driven architecture", "slug": "event-driven-architecture", "def": "Services reacting to published events, such as \"run completed\" or \"tool approved\", instead of calling each other directly.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#event-driven-architecture" }, { "name": "Event messaging", "slug": "event-messaging", "def": "Publishing and subscribing to events through a broker, with fan-out to many consumers.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#event-messaging" }, { "name": "Distributed coordination", "slug": "distributed-coordination", "def": "Getting nodes to agree on leaders, locks, membership or configuration, often through a consensus system.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#distributed-coordination" }, { "name": "Concurrency limits", "slug": "concurrency-limits", "def": "Caps on how many operations run at once per user, tenant, tool or dependency, protecting the systems downstream.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#concurrency-limits" }, { "name": "Work distribution", "slug": "work-distribution", "def": "Spreading tasks across workers fairly and efficiently.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#work-distribution" }, { "name": "Distributed locks", "slug": "distributed-locks", "def": "Locks held across processes or machines so only one actor works on a resource at a time.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#distributed-locks" }, { "name": "Leases", "slug": "leases", "def": "Time-limited locks or ownership grants that expire on their own if the holder dies, so work never stays stuck.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#leases" }, { "name": "Lease-based ownership", "slug": "lease-based-ownership", "def": "A worker owns a task only while it keeps renewing the lease. If it crashes, the lease lapses and another worker takes over.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#lease-based-ownership" }, { "name": "Work stealing", "slug": "work-stealing", "def": "Idle workers taking queued tasks from busy ones to even out the load.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#work-stealing" }, { "name": "Backpressure", "slug": "backpressure", "def": "Signalling upstream to slow down when downstream is overloaded, instead of letting queues grow without limit.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#backpressure" }, { "name": "Idempotency", "slug": "idempotency", "def": "The property that doing an operation twice has the same effect as doing it once. Essential wherever retries can happen.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#idempotency" }, { "name": "Idempotency keys", "slug": "idempotency-keys", "def": "Unique IDs attached to requests so the server can recognise and skip duplicates of an operation it has already done.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#idempotency-keys" }, { "name": "Transactional outbox", "slug": "transactional-outbox", "def": "Writing a database change and the event announcing it in the same transaction, then publishing from that outbox table, so the data and the events can never disagree.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#transactional-outbox" }, { "name": "Compensating actions", "slug": "compensating-actions", "def": "Steps that undo or offset an earlier action when a later step fails, as in the saga pattern, since true distributed transactions are rarely available.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#compensating-actions" }, { "name": "Exactly-once execution", "slug": "exactly-once-execution", "def": "The goal that each operation takes effect exactly once. Exactly-once delivery over a network is impossible, so systems combine at-least-once delivery with idempotency to get the same effect.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#exactly-once-execution" }, { "name": "At-least-once execution", "slug": "at-least-once-execution", "def": "A guarantee that an operation will run, possibly more than once. Safe only for idempotent operations.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#at-least-once-execution" }, { "name": "Fault tolerance", "slug": "fault-tolerance", "def": "Continuing to work correctly when components fail, through redundancy, retries and isolation.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#fault-tolerance" }, { "name": "Distributed recovery", "slug": "distributed-recovery", "def": "Bringing a distributed system back to a consistent state after a failure, for example by reassigning leases and replaying logs.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#distributed-recovery" }, { "name": "Resource allocation", "slug": "resource-allocation", "def": "Assigning CPU, memory, GPUs and sandbox slots to tasks according to their needs and the policy.", "url": "https://llm-atlas-7or.pages.dev/learn/backend/#resource-allocation" } ] }, { "n": 13, "slug": "security", "name": "Reliability, security & sandboxing", "tagline": "Letting an agent act in the world without letting it break things.", "track": "production", "intro": [ "An agent that runs code, browses the web or calls APIs is executing instructions that came from a model, and models can be wrong or manipulated by the content they read. The defence is layers: isolate where code runs, check every action against policy in code the model cannot talk its way past, and require a person for anything irreversible.", "Reliability is the same mindset applied to failure. Tools time out, networks drop, processes crash. Retries need idempotency, timeouts need cancellation, and anything with an external side effect needs to be safe to replay." ], "figure": { "title": "Defence in layers", "description": "A tool call travels from the model toward the host and must pass each boundary in turn. Pick an action to see where it is stopped: safe reads go through, risky actions wait for human approval, and anything the policy does not recognise is denied, because the gateway fails closed." }, "url": "https://llm-atlas-7or.pages.dev/learn/security/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/security.md", "terms": [ { "name": "Tool isolation", "slug": "tool-isolation", "def": "Running each tool so that a failure or compromise cannot affect the agent runtime, other tools or other users.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#tool-isolation" }, { "name": "Execution sandbox", "slug": "execution-sandbox", "def": "A restricted environment where agent-generated code and commands run, with limited filesystem, network and system access.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#execution-sandbox" }, { "name": "Sandboxed execution", "slug": "sandboxed-execution", "def": "Running untrusted code inside a sandbox. It should be the default for any code an LLM writes.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#sandboxed-execution" }, { "name": "Docker containers", "slug": "docker-containers", "def": "Lightweight isolated environments that share the host's kernel. Convenient for packaging tools, but the shared kernel makes them a weaker wall against hostile code.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#docker-containers" }, { "name": "MicroVMs", "slug": "microvms", "def": "Minimal virtual machines, such as Firecracker, that boot in milliseconds and give each sandbox its own kernel, a stronger boundary than containers.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#microvms" }, { "name": "Process isolation", "slug": "process-isolation", "def": "Separating work into OS processes with their own memory and privileges, so one crash or exploit stays contained.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#process-isolation" }, { "name": "Workspace isolation", "slug": "workspace-isolation", "def": "Giving each agent session its own working directory or volume so tasks cannot read or damage each other's files.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#workspace-isolation" }, { "name": "Filesystem boundaries", "slug": "filesystem-boundaries", "def": "Limits on which paths an agent can read or write, enforced with mounts, permissions or path checks.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#filesystem-boundaries" }, { "name": "Browser isolation", "slug": "browser-isolation", "def": "Running agent web browsing in a separate, disposable browser so pages cannot reach the user's real sessions or the host machine.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#browser-isolation" }, { "name": "Security policies", "slug": "security-policies", "def": "Rules that define what agents may do: allowed tools, domains and data, and which actions need approval.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#security-policies" }, { "name": "Capability security", "slug": "capability-security", "def": "Granting access by handing out specific, unforgeable capabilities, such as a token for one folder, instead of relying on broad ambient identity.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#capability-security" }, { "name": "Authorization", "slug": "authorization", "def": "Deciding whether an authenticated identity may perform a given action on a given resource.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#authorization" }, { "name": "Authentication", "slug": "authentication", "def": "Verifying who or what is making a request, for users and for agents alike.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#authentication" }, { "name": "Permission enforcement", "slug": "permission-enforcement", "def": "Checking permissions at the moment of action, in code the model cannot bypass, not just in the prompt.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#permission-enforcement" }, { "name": "Policy gateway", "slug": "policy-gateway", "def": "A central checkpoint that every tool call passes through to be judged against policy before it runs.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#policy-gateway" }, { "name": "Fail-closed security", "slug": "fail-closed-security", "def": "When a check cannot be completed, because of an error, a timeout or an unknown action, deny by default instead of allowing.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#fail-closed-security" }, { "name": "Audit logging", "slug": "audit-logging", "def": "Recording who did what, when and with which inputs, especially for actions with side effects.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#audit-logging" }, { "name": "Audit trails", "slug": "audit-trails", "def": "The tamper-evident sequence of audit records that lets an incident be reconstructed afterwards.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#audit-trails" }, { "name": "Human approval", "slug": "human-approval", "def": "Requiring a person to confirm sensitive actions, such as payments, deletions or outgoing messages, before they run.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#human-approval" }, { "name": "Human-in-the-loop", "slug": "human-in-the-loop", "def": "Designs where people review, correct or approve agent work at key points instead of granting full autonomy.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#human-in-the-loop" }, { "name": "Secrets management", "slug": "secrets-management", "def": "Storing and injecting API keys and credentials securely, through vaults and short-lived tokens, so they never end up in prompts or logs.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#secrets-management" }, { "name": "Credential encryption", "slug": "credential-encryption", "def": "Encrypting stored credentials at rest and in transit, with tightly controlled keys.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#credential-encryption" }, { "name": "Error handling", "slug": "error-handling", "def": "Catching, classifying and responding to failures: deciding whether to retry, tell the model, or abort.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#error-handling" }, { "name": "Retries", "slug": "retries", "def": "Repeating a failed operation with exponential backoff and jitter, only for transient failures and idempotent operations.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#retries" }, { "name": "Retry amplification", "slug": "retry-amplification", "def": "When retries at several layers multiply (three layers of three retries is 27 attempts) and turn a small outage into a self-inflicted overload.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#retry-amplification" }, { "name": "Timeouts", "slug": "timeouts", "def": "Upper limits on how long an operation may run before it is abandoned, applied to model calls, tools and whole runs.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#timeouts" }, { "name": "Cancellation", "slug": "cancellation", "def": "Stopping in-flight work cleanly when a user or parent task cancels, and passing that on to sub-tasks and tool processes.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#cancellation" }, { "name": "Crash recovery", "slug": "crash-recovery", "def": "Restoring service and in-progress work after a process or node crash, usually from checkpoints and durable logs.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#crash-recovery" }, { "name": "Failure handling", "slug": "failure-handling", "def": "The overall plan for failures: detect, contain, retry, compensate, escalate.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#failure-handling" }, { "name": "Replay safety", "slug": "replay-safety", "def": "Making sure that re-running steps during recovery does not repeat side effects, using idempotency keys and recorded results.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#replay-safety" }, { "name": "External side effects", "slug": "external-side-effects", "def": "Effects outside your own system, such as sending email, charging a card or writing to a third-party API, that cannot simply be rolled back.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#external-side-effects" }, { "name": "Rate limiting", "slug": "rate-limiting", "def": "Restricting how many requests or tokens a client may use per time window, protecting capacity and budgets.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#rate-limiting" }, { "name": "Resource limits", "slug": "resource-limits", "def": "Caps on CPU, memory, disk, processes, network and run time for sandboxes and agents.", "url": "https://llm-atlas-7or.pages.dev/learn/security/#resource-limits" } ] }, { "n": 14, "slug": "observability", "name": "Observability & performance", "tagline": "Seeing where the time, tokens and money actually go.", "track": "production", "intro": [ "An agent run is a tree of model calls, tool calls and waits, spread across services. Distributed tracing records that tree as timed spans, so a slow answer can be pinned on a long prefill, a sluggish tool or a queue that backed up.", "Averages hide most of the story. Users feel the slow tail, agents multiply it across every step, and GPU dashboards can say \"busy\" while the hardware does little useful work. Good observability measures percentiles, tokens and task outcomes, not just requests." ], "figure": { "title": "One agent run, traced", "description": "A trace waterfall of a single agent run: each bar is a span, nested under its parent, positioned by when it started. Hover or tap a span for details. Below it, the latency of 400 simulated runs shows why p95 and p99 matter more than the mean." }, "url": "https://llm-atlas-7or.pages.dev/learn/observability/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/observability.md", "terms": [ { "name": "Observability", "slug": "observability", "def": "Being able to understand what a system is doing inside from what it emits: traces, logs and metrics.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#observability" }, { "name": "Distributed tracing", "slug": "distributed-tracing", "def": "Following one request across services as a tree of timed spans, showing where its time went.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#distributed-tracing" }, { "name": "Execution traces", "slug": "execution-traces", "def": "Traces of an agent run: every model call, tool call and decision as a span with inputs, outputs and timing.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#execution-traces" }, { "name": "Trace correlation", "slug": "trace-correlation", "def": "Linking logs, metrics and spans through shared trace and span IDs, so one request can be followed everywhere.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#trace-correlation" }, { "name": "Structured logging", "slug": "structured-logging", "def": "Logging machine-readable records, usually JSON with consistent fields, instead of free-form text.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#structured-logging" }, { "name": "Metrics", "slug": "metrics", "def": "Numeric time series such as counters, gauges and histograms that show how a system behaves over time.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#metrics" }, { "name": "GPU telemetry", "slug": "gpu-telemetry", "def": "Hardware metrics from GPUs, including utilisation, memory, temperature, power, clocks, NVLink traffic and errors, usually gathered through DCGM.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#gpu-telemetry" }, { "name": "Performance profiling", "slug": "performance-profiling", "def": "Measuring where time and resources are spent in code, to find the bottlenecks.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#performance-profiling" }, { "name": "Inference profiling", "slug": "inference-profiling", "def": "Profiling a serving stack specifically: prefill against decode time, kernel time, scheduler overhead and batch composition.", "href": "/learn/inference/", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#inference-profiling" }, { "name": "Latency", "slug": "latency", "def": "How long an operation takes. For LLMs, time to first token, time per output token and end-to-end time should be tracked separately.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#latency" }, { "name": "End-to-end latency", "slug": "end-to-end-latency", "def": "The total time from a user's action to the complete result, including queues, every model call and every tool.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#end-to-end-latency" }, { "name": "p95 latency", "slug": "p95-latency", "def": "The value that 95 percent of requests come in under. Percentiles reveal what averages hide.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#p95-latency" }, { "name": "Tail latency", "slug": "tail-latency", "def": "High-percentile latency (p99, p99.9). In multi-step agents one slow step slows the whole run, so tails compound.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#tail-latency" }, { "name": "Throughput", "slug": "throughput", "def": "Work completed per unit of time: tokens per second, requests per second or tasks per hour.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#throughput" }, { "name": "Tokens per second", "slug": "tokens-per-second", "def": "Generation speed, either per stream (what one user sees) or summed across streams (what the server delivers).", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#tokens-per-second" }, { "name": "Requests per second", "slug": "requests-per-second", "def": "The rate of requests a system handles. For LLMs it means little without the token counts behind it.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#requests-per-second" }, { "name": "Task completion rate", "slug": "task-completion-rate", "def": "The share of agent tasks that finish successfully, often the metric users care about most.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#task-completion-rate" }, { "name": "Time to first token", "slug": "time-to-first-token", "def": "TTFT as an observed metric, from request arrival to the first streamed token, ideally broken down into queueing and prefill.", "href": "/learn/inference/#time-to-first-token-ttft", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#time-to-first-token" }, { "name": "Scheduling delay", "slug": "scheduling-delay", "def": "The time between a request being ready and actually being scheduled, in the inference engine or the orchestration layer.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#scheduling-delay" }, { "name": "Queue depth", "slug": "queue-depth", "def": "The number of items waiting in a queue, a leading indicator of overload.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#queue-depth" }, { "name": "GPU memory usage", "slug": "gpu-memory-usage", "def": "Memory consumed on each GPU by weights, KV cache, activations and fragmentation.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#gpu-memory-usage" }, { "name": "GPU utilization", "slug": "gpu-utilization", "def": "The share of time kernels are running on the GPU. Useful but coarse; pair it with multiprocessor occupancy and tensor-core activity.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#gpu-utilization" }, { "name": "KV-cache utilization", "slug": "kv-cache-utilization", "def": "The share of KV-cache blocks in use. Close to 100 percent means preemption and queueing are about to start.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#kv-cache-utilization" }, { "name": "Cost tracking", "slug": "cost-tracking", "def": "Attributing token, GPU and tool costs to users, features and runs.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#cost-tracking" }, { "name": "Token usage", "slug": "token-usage", "def": "Input, cached and output tokens per call, the main driver of LLM cost.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#token-usage" }, { "name": "Load testing", "slug": "load-testing", "def": "Driving synthetic traffic at a system to measure its capacity and find its breaking points, for example with k6.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#load-testing" }, { "name": "Benchmarking", "slug": "benchmarking", "def": "Measuring performance or quality under fixed, repeatable conditions to compare systems or versions.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#benchmarking" }, { "name": "Performance regressions", "slug": "performance-regressions", "def": "Changes that make a system slower or more expensive than before, ideally caught by benchmarks in CI.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#performance-regressions" }, { "name": "Bottleneck analysis", "slug": "bottleneck-analysis", "def": "Finding the one resource that limits throughput, whether GPU compute, memory bandwidth, a lock or a queue.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#bottleneck-analysis" }, { "name": "Capacity planning", "slug": "capacity-planning", "def": "Forecasting demand and provisioning enough resources ahead of time, using load tests and growth data.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#capacity-planning" }, { "name": "Agent evaluations", "slug": "agent-evaluations", "def": "Systematic tests of agent behaviour on representative tasks, scored by checks, rubrics or model graders.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#agent-evaluations" }, { "name": "Behavioral correctness", "slug": "behavioral-correctness", "def": "Whether an agent does the right thing: the right tools in the right order, the right outcome, and its constraints respected.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#behavioral-correctness" }, { "name": "Regression testing", "slug": "regression-testing", "def": "Re-running a fixed test suite after every change to catch behaviour that broke, including evaluations after prompt or model updates.", "url": "https://llm-atlas-7or.pages.dev/learn/observability/#regression-testing" } ] }, { "n": 15, "slug": "training", "name": "Model training & post-training", "tagline": "How a pile of text becomes a model that follows instructions and uses tools.", "track": "research", "intro": [ "Training is gradient descent at enormous scale. The model makes a prediction, a loss function measures how wrong it was, backpropagation works out how every parameter contributed, and an optimizer nudges them all to do slightly better. Repeat a few million times.", "Pretraining on trillions of tokens builds knowledge and fluency. Post-training then shapes behaviour: supervised fine-tuning on demonstrations teaches the chat format, and preference and reinforcement learning teach the model to be helpful, follow rules and use tools well." ], "figure": { "title": "Gradient descent, by hand", "description": "A ball rolling downhill on a loss surface. Each step follows the negative gradient, scaled by the learning rate. Too small and progress crawls; too large and it overshoots and zigzags. Momentum, the idea behind Adam's first moment, smooths the path. Drag the learning rate and press Train." }, "url": "https://llm-atlas-7or.pages.dev/learn/training/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/training.md", "terms": [ { "name": "Pretraining", "slug": "pretraining", "def": "The first and largest training stage: next-token prediction on trillions of tokens of web text, code and books. It gives the model its knowledge and language ability.", "href": "/learn/fundamentals/#next-token-prediction", "url": "https://llm-atlas-7or.pages.dev/learn/training/#pretraining" }, { "name": "Supervised fine-tuning (SFT)", "slug": "supervised-fine-tuning-sft", "def": "Training on curated prompt and response pairs so a base model learns to follow instructions and a chat format.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#supervised-fine-tuning-sft" }, { "name": "Post-training", "slug": "post-training", "def": "Everything after pretraining that shapes behaviour: SFT, preference optimisation (RLHF, DPO), reinforcement learning with verifiable rewards and safety training.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#post-training" }, { "name": "Tool-use training", "slug": "tool-use-training", "def": "Fine-tuning on trajectories that contain tool calls and results, and reinforcement learning in tool environments, so models learn when and how to call tools.", "href": "/learn/tool-calling/", "url": "https://llm-atlas-7or.pages.dev/learn/training/#tool-use-training" }, { "name": "Training examples", "slug": "training-examples", "def": "The individual inputs and targets, or whole trajectories, a model learns from. In post-training, quality and diversity matter more than sheer count.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#training-examples" }, { "name": "Learning from demonstrations", "slug": "learning-from-demonstrations", "def": "Training a model to imitate example behaviour shown by people or stronger models (imitation learning). SFT is a form of it.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#learning-from-demonstrations" }, { "name": "Gradient descent", "slug": "gradient-descent", "def": "The optimisation method behind all of it: compute how the loss changes with each parameter (the gradient) and move every parameter a small step in the opposite direction.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#gradient-descent" }, { "name": "Backpropagation", "slug": "backpropagation", "def": "The algorithm that computes the gradient for every parameter efficiently, by applying the chain rule backwards through the network.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#backpropagation" }, { "name": "Loss functions", "slug": "loss-functions", "def": "Functions that measure how wrong a prediction is. LLM pretraining uses cross-entropy on the next token.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#loss-functions" }, { "name": "Optimizers", "slug": "optimizers", "def": "Algorithms that turn gradients into parameter updates, adding momentum, adaptive step sizes and weight decay, such as SGD, AdamW and Muon.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#optimizers" }, { "name": "Adam optimizer", "slug": "adam-optimizer", "def": "An optimiser that keeps running averages of each parameter's gradient and squared gradient, giving every parameter its own adaptive step size. AdamW, with decoupled weight decay, is the standard for LLMs.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#adam-optimizer" }, { "name": "Training datasets", "slug": "training-datasets", "def": "The text, code and trajectories used for training. Filtering, deduplication and mixing ratios shape the result as much as size does.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#training-datasets" }, { "name": "Model behavior", "slug": "model-behavior", "def": "How a model actually responds, including style, helpfulness, refusals and tool use. It is shaped mainly in post-training.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#model-behavior" }, { "name": "Model capabilities", "slug": "model-capabilities", "def": "What a model can do, such as reasoning, coding, languages, long context and tool use. Mostly built in pretraining and drawn out in post-training.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#model-capabilities" }, { "name": "Training feedback", "slug": "training-feedback", "def": "The signals used to improve a model: human preferences, reward models, verifiers or the results of running code.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#training-feedback" }, { "name": "Model evaluation", "slug": "model-evaluation", "def": "Measuring a model's quality on held-out tasks, safety tests and real-world use.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#model-evaluation" }, { "name": "Benchmark evaluation", "slug": "benchmark-evaluation", "def": "Scoring a model on standard benchmarks such as MMLU, GPQA, SWE-bench and MATH-500. Comparable across models, but contamination and overfitting limit what a score means.", "url": "https://llm-atlas-7or.pages.dev/learn/training/#benchmark-evaluation" } ] }, { "n": 16, "slug": "interpretability", "name": "Weight-space programming & interpretability", "tagline": "Reading and editing what a network has learned, directly in its weights.", "track": "research", "intro": [ "A weight matrix is not an opaque blob. It can be decomposed into directions: things a layer reads from the residual stream and things it writes back, each with a strength. Singular value decomposition makes that structure explicit, and a handful of directions often carry most of a matrix's effect.", "Once directions can be found, they can be changed. Weight-space programming edits parameters directly to install, remove or soften a behaviour, such as a repetitive reasoning loop, and then checks with regression evaluations that nothing else broke." ], "figure": { "title": "A matrix, one direction at a time", "description": "A 16×16 weight matrix rebuilt from its singular components. The rank slider keeps only the strongest k directions; most of the structure comes back with very few. Click a singular value to ablate that direction and see exactly what it contributed, the basic move behind weight editing." }, "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/interpretability.md", "terms": [ { "name": "Weight-space programming", "slug": "weight-space-programming", "def": "Changing a model's behaviour by writing to its weights directly, as an alternative to prompting or to fine-tuning on data.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#weight-space-programming" }, { "name": "Weight-space reading", "slug": "weight-space-reading", "def": "Analysing weights directly, without running inputs, to find structure such as dominant directions or what a neuron reads and writes.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#weight-space-reading" }, { "name": "Weight-space writing", "slug": "weight-space-writing", "def": "Editing parameter values to install, remove or change a behaviour.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#weight-space-writing" }, { "name": "Weight-space editing", "slug": "weight-space-editing", "def": "Targeted modification of specific weights, ideally changing one behaviour and leaving everything else intact.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#weight-space-editing" }, { "name": "Direct weight modification", "slug": "direct-weight-modification", "def": "Changing parameter tensors by hand or by algorithm, without gradient training on a dataset.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#direct-weight-modification" }, { "name": "Neural network parameter inspection", "slug": "neural-network-parameter-inspection", "def": "Examining parameter tensors: their statistics, norms, spectra and how they connect components.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#neural-network-parameter-inspection" }, { "name": "Numerical weight edits", "slug": "numerical-weight-edits", "def": "Precise arithmetic changes to weights, such as adding a rank-one update or scaling one direction.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#numerical-weight-edits" }, { "name": "Weight matrices", "slug": "weight-matrices", "def": "The 2D parameter arrays of each layer. Interpretability treats them as maps that read from and write to directions in the residual stream.", "href": "/learn/fundamentals/#weight-matrices", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#weight-matrices" }, { "name": "Tensor inspection", "slug": "tensor-inspection", "def": "Looking at the values, shapes and distributions of weights or activations, often visually.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#tensor-inspection" }, { "name": "Matrix properties", "slug": "matrix-properties", "def": "Features of a weight matrix such as rank, norm, condition number and singular-value spectrum, which hint at its role and stability.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#matrix-properties" }, { "name": "Activation patterns", "slug": "activation-patterns", "def": "Which neurons or directions fire for which inputs. Recording them over many prompts reveals features.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#activation-patterns" }, { "name": "Singular vectors", "slug": "singular-vectors", "def": "The input and output directions found by SVD. A matrix maps each right singular vector onto its left singular vector, scaled by the singular value.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#singular-vectors" }, { "name": "Singular value decomposition (SVD)", "slug": "singular-value-decomposition-svd", "def": "Factoring a matrix as W = UΣVᵀ: sets of orthogonal directions and the strength of each. Keeping only the top components gives the best possible low-rank approximation.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#singular-value-decomposition-svd" }, { "name": "Behavioral directions", "slug": "behavioral-directions", "def": "Directions in activation space that line up with a behaviour, such as refusal, sentiment or truthfulness. Adding or removing them steers the model.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#behavioral-directions" }, { "name": "MLP weight editing", "slug": "mlp-weight-editing", "def": "Editing FFN weights, which partly act as key-value memories, to change stored associations, as in methods like ROME and MEMIT.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#mlp-weight-editing" }, { "name": "Mechanistic interpretability", "slug": "mechanistic-interpretability", "def": "Reverse-engineering the algorithms a network implements, down to individual features and circuits.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#mechanistic-interpretability" }, { "name": "Model editing", "slug": "model-editing", "def": "Changing specific facts or behaviours of a trained model with as few side effects as possible.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#model-editing" }, { "name": "Targeted model intervention", "slug": "targeted-model-intervention", "def": "Changing activations or weights at one specific location, to test a causal hypothesis or to fix a behaviour.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#targeted-model-intervention" }, { "name": "Parameter-efficient modifications", "slug": "parameter-efficient-modifications", "def": "Changing a model by touching only a few parameters, such as LoRA adapters, rank-one edits or bias tweaks.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#parameter-efficient-modifications" }, { "name": "Failure-mode localization", "slug": "failure-mode-localization", "def": "Finding which layers, heads or directions cause a particular failure, so it can be fixed precisely.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#failure-mode-localization" }, { "name": "Repetitive reasoning loops", "slug": "repetitive-reasoning-loops", "def": "A failure mode where a reasoning model keeps repeating the same thoughts or tokens without finishing. It can be traced to specific directions and reduced by editing them or by changing decoding settings.", "href": "/learn/decoding/#repetition-penalty", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#repetitive-reasoning-loops" }, { "name": "Automated neural network engineering", "slug": "automated-neural-network-engineering", "def": "Using tools, often AI agents, to inspect, modify and test networks automatically rather than by hand.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#automated-neural-network-engineering" }, { "name": "MATH-500 benchmark", "slug": "math-500-benchmark", "def": "A 500-problem subset of the MATH competition dataset, widely used to measure mathematical reasoning and a handy check that an edit did not hurt capability.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#math-500-benchmark" }, { "name": "Model regression evaluation", "slug": "model-regression-evaluation", "def": "Checking that an edited or updated model has not lost performance on tasks it used to handle.", "url": "https://llm-atlas-7or.pages.dev/learn/interpretability/#model-regression-evaluation" } ] }, { "n": 17, "slug": "stack", "name": "Infrastructure & development tools", "tagline": "The tools a modern agent and inference stack is actually built with.", "track": "research", "intro": [ "No single product covers the whole path from a user's click to a GPU kernel. A real system strings together a web framework, a database and a message broker, a workflow engine, an agent framework, an inference engine, and the observability tools to watch it all.", "The map below groups the tools in this guide by the layer they live in. Many are interchangeable within a layer, and most of the hard engineering happens at the seams between layers." ], "figure": { "title": "The stack, layer by layer", "description": "From the user-facing app at the top down to GPU kernels at the bottom. Select any tool to read what it does." }, "url": "https://llm-atlas-7or.pages.dev/learn/stack/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/stack.md", "terms": [ { "name": "ASP.NET Core", "slug": "asp-net-core", "def": "Microsoft's cross-platform, high-performance framework for building web APIs and services in C#.", "layer": "Apps & APIs", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#asp-net-core" }, { "name": "C# / .NET", "slug": "csharp-net", "def": "Microsoft's statically typed language and the runtime and libraries around it, a common choice for enterprise backends.", "layer": "Apps & APIs", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#csharp-net" }, { "name": "Entity Framework Core", "slug": "entity-framework-core", "def": "The .NET object-relational mapper for working with databases through C# classes and LINQ queries.", "layer": "Data & messaging", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#entity-framework-core" }, { "name": "PostgreSQL", "slug": "postgresql", "def": "A widely used open-source relational database known for correctness, extensions and JSON support. With the pgvector extension it also stores embeddings.", "layer": "Data & messaging", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#postgresql" }, { "name": "JSONB", "slug": "jsonb", "def": "PostgreSQL's binary JSON column type, which can be indexed and queried. Handy for flexible agent state and tool payloads.", "layer": "Data & messaging", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#jsonb" }, { "name": "Redis", "slug": "redis", "def": "An in-memory data store used for caching, rate limiting, locks, queues and pub/sub.", "layer": "Data & messaging", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#redis" }, { "name": "NATS JetStream", "slug": "nats-jetstream", "def": "The persistence layer of the NATS messaging system, adding durable streams, replay and acknowledged consumers.", "layer": "Data & messaging", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#nats-jetstream" }, { "name": "Temporal", "slug": "temporal", "def": "A durable execution platform. Workflows are written as ordinary code, their progress is recorded, and they survive crashes by resuming through deterministic replay.", "layer": "Data & messaging", "href": "/learn/context-memory/#deterministic-replay", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#temporal" }, { "name": "Docker", "slug": "docker", "def": "Tooling to build and run containers, packaging an application together with its dependencies.", "layer": "Infrastructure & delivery", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#docker" }, { "name": "Docker Compose", "slug": "docker-compose", "def": "A tool for defining and running multi-container setups from one YAML file, mostly for local development.", "layer": "Infrastructure & delivery", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#docker-compose" }, { "name": "Kubernetes", "slug": "kubernetes", "def": "The standard container orchestrator for deploying, scaling and healing services across a cluster, GPU workloads included.", "layer": "Infrastructure & delivery", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#kubernetes" }, { "name": "Terraform", "slug": "terraform", "def": "An infrastructure-as-code tool that declares cloud resources in configuration files and applies changes reproducibly.", "layer": "Infrastructure & delivery", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#terraform" }, { "name": "GitHub Actions", "slug": "github-actions", "def": "GitHub's CI/CD service for running builds, tests and deployments on every change.", "layer": "Infrastructure & delivery", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#github-actions" }, { "name": "OpenTelemetry", "slug": "opentelemetry", "def": "The vendor-neutral standard and SDKs for producing traces, metrics and logs, with emerging conventions for LLM and agent spans.", "layer": "Observability", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#opentelemetry" }, { "name": "Prometheus", "slug": "prometheus", "def": "A time-series database and monitoring system that scrapes metrics and evaluates alert rules.", "layer": "Observability", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#prometheus" }, { "name": "Grafana", "slug": "grafana", "def": "Dashboards and visualisation across metrics, logs and traces.", "layer": "Observability", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#grafana" }, { "name": "Grafana Tempo", "slug": "grafana-tempo", "def": "Grafana's distributed tracing backend for storing and querying traces at scale.", "layer": "Observability", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#grafana-tempo" }, { "name": "Grafana Loki", "slug": "grafana-loki", "def": "Grafana's log aggregation system. It indexes labels rather than full text, which keeps it cheap to run.", "layer": "Observability", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#grafana-loki" }, { "name": "NVIDIA DCGM Exporter", "slug": "nvidia-dcgm-exporter", "def": "Exposes NVIDIA Data Center GPU Manager metrics, such as utilisation, memory, temperature and errors, to Prometheus.", "layer": "Observability", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#nvidia-dcgm-exporter" }, { "name": "xUnit", "slug": "xunit", "def": "A popular unit-testing framework for .NET.", "layer": "Testing", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#xunit" }, { "name": "Testcontainers", "slug": "testcontainers", "def": "A library that starts real dependencies, such as databases and brokers, in throwaway containers during tests.", "layer": "Testing", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#testcontainers" }, { "name": "k6", "slug": "k6", "def": "A load-testing tool whose scenarios are written in JavaScript, good for measuring API capacity and latency percentiles.", "layer": "Testing", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#k6" }, { "name": "Next.js", "slug": "next-js", "def": "A React framework for web apps with server rendering, static export and file-based routing. This site is built with it.", "layer": "Apps & APIs", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#next-js" }, { "name": "TypeScript", "slug": "typescript", "def": "JavaScript with static types, the default for modern web front ends and many agent frameworks.", "layer": "Apps & APIs", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#typescript" }, { "name": "shadcn/ui", "slug": "shadcn-ui", "def": "A collection of copy-in React components built on headless primitives and Tailwind CSS. This site uses it on top of Base UI.", "layer": "Apps & APIs", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#shadcn-ui" }, { "name": "Python", "slug": "python", "def": "The main language of machine-learning research, training and most inference tooling.", "layer": "Apps & APIs", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#python" }, { "name": "vLLM", "slug": "vllm", "def": "A popular open-source LLM inference engine that introduced PagedAttention, with continuous batching, prefix caching and an OpenAI-compatible server.", "layer": "Inference engines", "href": "/learn/batching/#continuous-batching", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#vllm" }, { "name": "SGLang", "slug": "sglang", "def": "An open-source serving engine with RadixAttention for automatic prefix sharing, fast structured outputs and strong support for large MoE models.", "layer": "Inference engines", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#sglang" }, { "name": "llama.cpp", "slug": "llama-cpp", "def": "A C/C++ inference engine for running quantized models (GGUF files) on CPUs, Apple Silicon and consumer GPUs.", "layer": "Inference engines", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#llama-cpp" }, { "name": "TensorRT-LLM", "slug": "tensorrt-llm", "def": "NVIDIA's library for heavily optimised LLM inference on NVIDIA GPUs, with compiled kernels, FP8 and FP4, and in-flight batching.", "layer": "Inference engines", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#tensorrt-llm" }, { "name": "llm-d", "slug": "llm-d", "def": "An open-source, Kubernetes-native distributed inference stack built around vLLM, adding cache-aware routing and prefill/decode disaggregation.", "layer": "Inference engines", "href": "/learn/distributed/#prefill-decode-disaggregation", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#llm-d" }, { "name": "Microsoft Agent Framework", "slug": "microsoft-agent-framework", "def": "Microsoft's open-source SDK for building agents and multi-agent workflows in .NET and Python, the successor to Semantic Kernel and AutoGen.", "layer": "Agent frameworks", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#microsoft-agent-framework" }, { "name": "LangGraph", "slug": "langgraph", "def": "A library from the LangChain team for building agents as stateful graphs, with persistence, checkpoints and human-in-the-loop steps.", "layer": "Agent frameworks", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#langgraph" }, { "name": "CrewAI", "slug": "crewai", "def": "A Python framework for orchestrating teams of role-based agents.", "layer": "Agent frameworks", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#crewai" }, { "name": "Pydantic AI", "slug": "pydantic-ai", "def": "A Python agent framework from the Pydantic team, built around type-safe tool definitions and validated structured outputs.", "layer": "Agent frameworks", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#pydantic-ai" }, { "name": "Google ADK", "slug": "google-adk", "def": "Google's Agent Development Kit, an open-source framework for building, evaluating and deploying agents, integrated with Gemini and Vertex AI.", "layer": "Agent frameworks", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#google-adk" }, { "name": "Mastra", "slug": "mastra", "def": "A TypeScript framework for building agents, workflows and retrieval pipelines.", "layer": "Agent frameworks", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#mastra" }, { "name": "CopilotKit", "slug": "copilotkit", "def": "CopilotKit is an open-source toolkit for putting agents inside React apps, with in-app chat, shared state and generative UI. Its team created the AG-UI protocol.", "layer": "Agent frameworks", "href": "/learn/tool-calling/#ag-ui-protocol", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#copilotkit" }, { "name": "OpenClaw", "slug": "openclaw", "def": "An open-source, self-hosted personal AI agent that runs on your own machine and is driven through chat apps, extended with tools and skills.", "layer": "Agent frameworks", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#openclaw" }, { "name": "Hermes Agent", "slug": "hermes-agent", "def": "An open-source agent from Nous Research, built around its Hermes models, with persistent memory, skills and tool use.", "layer": "Agent frameworks", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#hermes-agent" }, { "name": "Megatron-LM", "slug": "megatron-lm", "def": "NVIDIA's library for training very large transformers with tensor, pipeline, sequence and expert parallelism.", "layer": "Models & training", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#megatron-lm" }, { "name": "Qwen", "slug": "qwen", "def": "Alibaba's family of open-weight models, dense and MoE, covering text, code and vision. Many of its releases are in the gallery.", "layer": "Models & training", "href": "/models/?lab=Alibaba+Cloud", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#qwen" }, { "name": "Llama", "slug": "llama", "def": "Meta's open-weight model family, whose 2023 release kick-started the open-weight ecosystem.", "layer": "Models & training", "href": "/models/?lab=Meta", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#llama" }, { "name": "DeepSeek", "slug": "deepseek", "def": "The Chinese lab behind DeepSeek-V3 and R1, known for multi-head latent attention, fine-grained MoE and very efficient training.", "layer": "Models & training", "href": "/models/?lab=DeepSeek", "concepts": [ "mla" ], "url": "https://llm-atlas-7or.pages.dev/learn/stack/#deepseek" }, { "name": "CUDA", "slug": "cuda", "def": "NVIDIA's GPU programming platform, the foundation under nearly every training and inference stack.", "layer": "Kernels", "href": "/learn/gpu/#cuda", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#cuda" }, { "name": "Triton", "slug": "triton", "def": "The Python-like GPU kernel language and compiler used for custom fused kernels in PyTorch and inference engines (not to be confused with NVIDIA's Triton Inference Server).", "layer": "Kernels", "href": "/learn/gpu/#triton-language", "url": "https://llm-atlas-7or.pages.dev/learn/stack/#triton" } ] }, { "n": 18, "slug": "research", "name": "Advanced systems research questions", "tagline": "Open problems where agents and inference infrastructure meet.", "track": "research", "intro": [ "Today agent runtimes and inference engines mostly treat each other as black boxes. The runtime sends independent requests; the engine schedules them without knowing that ten of them belong to the same task, share a long prefix, and that one sits on the critical path.", "Many open questions come from closing that gap: scheduling agents and inference together, routing by context and cost, and measuring success in tasks completed rather than tokens served. A useful lens is to think of the agent runtime as an operating system." ], "figure": { "title": "The agent runtime as an operating system", "description": "Classic operating-system ideas map surprisingly well onto agent infrastructure. Select a row to see the analogy and the research question it raises." }, "url": "https://llm-atlas-7or.pages.dev/learn/research/", "markdownUrl": "https://llm-atlas-7or.pages.dev/learn/research.md", "terms": [ { "name": "Inference-aware agent scheduling", "slug": "inference-aware-agent-scheduling", "def": "Scheduling agent steps with knowledge of the inference layer, for example grouping calls that share a prefix, or holding back low-priority agents when GPUs are saturated.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#inference-aware-agent-scheduling" }, { "name": "Agent-aware inference serving", "slug": "agent-aware-inference-serving", "def": "Inference engines that understand agent workloads: keeping an agent's KV cache warm between steps, prioritising calls on a task's critical path, anticipating the next call.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#agent-aware-inference-serving" }, { "name": "Joint agent/inference scheduling", "slug": "joint-agent-inference-scheduling", "def": "Optimising agent orchestration and model serving together instead of treating the inference API as a black box.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#joint-agent-inference-scheduling" }, { "name": "GPU-aware orchestration", "slug": "gpu-aware-orchestration", "def": "Orchestration that considers GPU locality, memory and load when placing agent work and model calls.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#gpu-aware-orchestration" }, { "name": "Multi-tenant agent infrastructure", "slug": "multi-tenant-agent-infrastructure", "def": "Shared platforms running many organisations' agents with isolation, fairness, quotas and per-tenant costs.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#multi-tenant-agent-infrastructure" }, { "name": "Bursty agent workloads", "slug": "bursty-agent-workloads", "def": "Agents produce sudden bursts of model and tool calls, from fan-out and parallel sub-agents, unlike the steadier traffic of chat.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#bursty-agent-workloads" }, { "name": "Agent workload characterization", "slug": "agent-workload-characterization", "def": "Measuring what agent workloads really look like: number of calls, prompt reuse, tool latency, context growth and branching.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#agent-workload-characterization" }, { "name": "Agent concurrency optimization", "slug": "agent-concurrency-optimization", "def": "Running independent agent steps and tool calls in parallel, and tuning how many run at once to maximise throughput without overload.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#agent-concurrency-optimization" }, { "name": "Task-throughput optimization", "slug": "task-throughput-optimization", "def": "Optimising completed tasks per unit of time or money, rather than tokens per second, as the real objective.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#task-throughput-optimization" }, { "name": "Cross-layer optimization", "slug": "cross-layer-optimization", "def": "Coordinated optimisations across the app, agent runtime, inference engine and hardware that no single layer can achieve alone.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#cross-layer-optimization" }, { "name": "Scheduling fairness", "slug": "scheduling-fairness", "def": "Making sure tenants and agents get fair shares of capacity, so one heavy user's bursts cannot starve everyone else.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#scheduling-fairness" }, { "name": "Cost-aware model routing", "slug": "cost-aware-model-routing", "def": "Sending each request to the cheapest model likely to handle it well, and escalating to larger models only when needed.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#cost-aware-model-routing" }, { "name": "Model placement", "slug": "model-placement", "def": "Deciding which models run on which GPUs, and how many replicas each gets, given demand and memory.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#model-placement" }, { "name": "Accelerator placement", "slug": "accelerator-placement", "def": "Matching workloads to accelerator types, such as H100s, consumer GPUs or TPUs, by performance per unit of cost.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#accelerator-placement" }, { "name": "Resource-aware scheduling", "slug": "resource-aware-scheduling", "def": "Scheduling with explicit knowledge of what each task needs: memory, KV-cache size, GPU type and sandbox capacity.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#resource-aware-scheduling" }, { "name": "Context-aware routing", "slug": "context-aware-routing", "def": "Routing requests by what is in their context, such as cached prefixes, length or modality, to the best-suited backend.", "href": "/learn/distributed/#cache-aware-routing", "url": "https://llm-atlas-7or.pages.dev/learn/research/#context-aware-routing" }, { "name": "Agent execution profiling", "slug": "agent-execution-profiling", "def": "Profiling whole agent runs to see how time splits between model calls, tools, queueing and overhead.", "href": "/learn/observability/", "url": "https://llm-atlas-7or.pages.dev/learn/research/#agent-execution-profiling" }, { "name": "Dynamic admission control", "slug": "dynamic-admission-control", "def": "Adjusting admission thresholds in real time from load signals, such as accepting fewer new agent runs while KV-cache utilisation is high.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#dynamic-admission-control" }, { "name": "Compute resource contention", "slug": "compute-resource-contention", "def": "Workloads competing for the same GPUs, memory bandwidth or CPUs and slowing each other down.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#compute-resource-contention" }, { "name": "Inference cost optimization", "slug": "inference-cost-optimization", "def": "Lowering the cost per token or per task through batching, caching, quantization, routing and right-sized hardware.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#inference-cost-optimization" }, { "name": "Distributed agent benchmarking", "slug": "distributed-agent-benchmarking", "def": "Benchmarking agent systems at scale on realistic, concurrent, multi-tenant workloads instead of single runs.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#distributed-agent-benchmarking" }, { "name": "Agent-runtime operating-system analogy", "slug": "agent-runtime-operating-system-analogy", "def": "Viewing the agent runtime as an operating system: the model as the CPU, the context window as RAM, tools as system calls, the scheduler deciding which agent runs, and context compaction as paging.", "url": "https://llm-atlas-7or.pages.dev/learn/research/#agent-runtime-operating-system-analogy" } ] } ] }