{ "type": "new-runtime-knowledge-guide", "version": 1, "generated_at": "2026-07-23", "volume": 1, "slug": "llm-pipeline-patterns", "title": "LLM Pipeline Patterns", "description": "A comprehensive catalog of steps and tricks for building production systems based on language models. Each pattern is atomically parsed.", "track": { "slug": "agent-systems", "title": "Agent Systems", "route": "https://newruntime.com/learn/tracks/agent-systems/", "position": 1 }, "counts": { "sections": 10, "cards": 47 }, "routes": { "html": "https://newruntime.com/learn/llm-pipeline-patterns/", "json": "https://newruntime.com/learn/llm-pipeline-patterns.json" }, "sections": [ { "number": "01", "title": "Input Processing", "intro": "", "cards": [ { "number": "01", "title": "Query Paraphrase / Rewrite", "tag": "input", "description": "Before sending a request to LLM or a search engine, reformulate it in a different way. This compensates for inaccurate user formulations and aligns the semantics to the index vector space.", "blocks": [ { "kind": "code", "label": "How it works", "language": "text", "value": "→ user: \"why does my back hurt\"\n→ LLM rewrite: “causes of low back pain, risk factors”\n→ search by reformulated query\n→ results are combined with the original context" }, { "kind": "list", "label": "Applicable", "items": [ "RAG system", "knowledge base search", "chatbots" ] } ] }, { "number": "02", "title": "Chunking Strategy", "tag": "input", "description": "Breaking a long document into chunks for indexing or transferring into context. Strategies: fixed-size, recursive semantic, sentence-based, paragraph-based, sliding window with overlap.", "blocks": [ { "kind": "code", "label": "Key idea", "language": "text", "value": "overlap = 20% // do not lose context at the junction of chunks\nchunk size = 512 tokens // accuracy/recall trade-off\nmetadata = {source, page, section} // for filtering" }, { "kind": "list", "label": "Applicable", "items": [ "RAG", "document QA", "summarization of long texts" ] } ] }, { "number": "03", "title": "Context Stuffing / Window Packing", "tag": "input", "description": "Maximum filling of the context window with relevant information from different sources: retrieved chunks + metadata + examples + history. The goal is to convey everything the model needs in one call.", "blocks": [ { "kind": "code", "label": "Prompt structure", "language": "text", "value": "[system_persona] [task_description]\n[retrieved_chunks × N]\n[few_shot_examples × K]\n[conversation_history]\n[user_query]" }, { "kind": "list", "label": "Applicable", "items": [ "minimizing the number of API calls", "cost reduction" ] } ] }, { "number": "04", "title": "HyDE — Hypothetical Document Embedding", "tag": "retrieval", "description": "Instead of searching by raw query, ask LLM to generate a hypothetical answer, and then search by its embedding. This works because the answer embedding is semantically closer to the actual documents than the question embedding.", "blocks": [ { "kind": "code", "label": "How it works", "language": "text", "value": "query: \"how does the transformer work?\"\n↓ LLM(query) → hypothetical answer ~200 words\n↓ embed(hypothetical_answer)\n↓vector_search(embedding)\n↓ real relevant documents" }, { "kind": "list", "label": "Applicable", "items": [ "technical search", "when the request is too short" ] } ] } ] }, { "number": "02", "title": "Prompt Engineering", "intro": "", "cards": [ { "number": "05", "title": "Chain-of-Thought (CoT)", "tag": "prompt", "description": "Explicitly instructing the model to “think out loud” before answering. The model generates intermediate steps of reasoning, which reduces errors on complex tasks - especially mathematics, logic, and multi-step inferences.", "blocks": [ { "kind": "code", "label": "Mechanism", "language": "sql", "value": "Prompt: \"Think step by step before answering\"\n→ the model reveals reasoning in its response\n→ the final answer follows from the steps derived\n\nOptions:\n zero-shot CoT: add \"Let's think step by step\"\n few-shot CoT: show examples with reasoning\n invisible CoT: ... tags (hidden)" }, { "kind": "list", "label": "Applicable", "items": [ "mathematics", "diagnostics", "planning", "debugging" ] } ] }, { "number": "06", "title": "Few-shot / Dynamic Few-shot", "tag": "prompt", "description": "Passing input→output examples directly to the prompt. Dynamic version - examples are selected from the repository through semantic search, relevant to the current request. This is in-context learning without fine-tuning.", "blocks": [ { "kind": "code", "label": "Dynamic Few-shot Pipeline", "language": "text", "value": "query → embed(query)\n → knn_search(examples_store, k=3)\n → selected examples in prompt\n → LLM(system + examples + query)" }, { "kind": "list", "label": "Applicable", "items": [ "structured output", "classification", "template generation" ] } ] }, { "number": "07", "title": "Meta-Prompting / Prompt Generation", "tag": "prompt", "description": "LLM generates or optimizes a prompt for another LLM call. It is used in automatic optimization of prompts (APE, DSPy-style), where the model itself offers improvements for the task.", "blocks": [ { "kind": "code", "label": "How it works", "language": "text", "value": "task + examples of failures\n→ Meta-LLM: \"write the best prompt for X\"\n→ generated prompt\n→ Worker-LLM(generated_prompt)\n→ evaluation → iteration" }, { "kind": "list", "label": "Applicable", "items": [ "auto-optimization", "DSPy", "prompt versioning" ] } ] }, { "number": "08", "title": "Persona / Role Injection", "tag": "prompt", "description": "Assign a role to system prompt to control style, tone, level of expertise, and output format. Activates the desired distribution of tokens in the model’s weights - it “switches the register”.", "blocks": [ { "kind": "code", "label": "Example", "language": "text", "value": "System: \"You are a senior data engineer with 10 years of experience\nin industrial analytics. Answer technically\nno water, use specific numbers.\"\n→ the model activates technical vocabulary and style" }, { "kind": "list", "label": "Applicable", "items": [ "tone control", "domain expertise", "products with UX" ] } ] }, { "number": "09", "title": "ReAct Pattern", "tag": "prompt", "description": "Interleaved reasoning + action. The model alternates between \"Thought: [reasoning]\" and \"Action: [calling a tool]\" and \"Observation: [result]\". This allows the agent to adapt to the results of the tools on the fly.", "blocks": [ { "kind": "code", "label": "Cycle", "language": "text", "value": "Thought: you need to check the current exchange rate\nAction: search(\"USD RUB exchange rate today\")\nObservation: 1 USD = 91.3 RUB (03/09/2025)\nThought: now I can calculate the amount\nAction: calc(1500 * 91.3)\nObservation: 136950\nAnswer: 1500 USD = 136,950 RUB" }, { "kind": "list", "label": "Applicable", "items": [ "agents with tools", "search + calculations" ] } ] } ] }, { "number": "03", "title": "Fan-out / Fan-in / Ensemble", "intro": "", "cards": [ { "number": "10", "title": "Fan-out - parallel independent calls", "tag": "parallelism", "description": "One incoming request launches N parallel LLM calls with different prompts, configurations, or subtasks. All calls are independent - there are no dependencies between them. The main goal is either to cover different aspects of the problem, or to collect a variety of answers for subsequent selection of the best one.", "blocks": [ { "kind": "code", "label": "Diagram", "language": "text", "value": " ┌─── LLM(prompt_A) ──→ result_A\nquery ──→ router ──┼─── LLM(prompt_B) ──→ result_B\n └─── LLM(prompt_C) ──→ result_C\n ↓↓↓ fan-in ↓↓↓\n aggregator(A, B, C) → final_output" }, { "kind": "list", "label": "When to use", "items": [ "generation of options", "covering different points of view", "parallel analysis of several documents", "A/B testing of prompts at runtime" ] } ] }, { "number": "11", "title": "Fan-in - aggregation of results", "tag": "aggregation", "description": "Assembling the results from N parallel calls into one final response. Strategies: concatenation, map-reduce summarization, voting, LLM synthesis.", "blocks": [ { "kind": "code", "label": "Fan-in strategies", "language": "sql", "value": "concat: just concatenate\nvote: by majority vote\nreduce: LLM(summarize([A,B,C]))\nscore+pick: select the best by metric\nmerge: LLM(merge([A,B,C]))" }, { "kind": "list", "label": "Applicable", "items": [ "ensemble of responses", "summary reports" ] } ] }, { "number": "12", "title": "Rerank - reranking candidates", "tag": "ranking", "description": "Two-stage search/generation: in the first stage, many candidates are quickly received (recall); in the second, an expensive model re-evaluates and rearranges them in descending order of relevance (precision). Key insight: fast retriever (BM25, vector search) can find, but ranks poorly. A slow reranker (cross-encoder, LLM) can accurately compare query-document pairs because it sees both at once - this is a fundamentally different type of attention.", "blocks": [ { "kind": "code", "label": "Diagram", "language": "text", "value": "Why is reranker more accurate than vector search?\n\nbi-encoder: embed(query) embed(doc) ← independent, fast, ~100ms\ncross-encoder: LLM([query; doc]) → score ← together, slowly, accurately\n\nPipeline:\nquery → fast_retriever (top-100 docs)\n → reranker (top-5 out of 100)\n → LLM (generation according to top-5)" }, { "kind": "list", "label": "Applicable", "items": [ "RAG with high precision requirements", "search engines", "choosing the best of N LLM options", "recommendations" ] } ] }, { "number": "13", "title": "Self-consistency / Majority Voting", "tag": "ensemble", "description": "Run the same prompt N times with temperature > 0, get different reasoning paths, take the answer by majority vote. Statistically more reliable than a single greedy call.", "blocks": [ { "kind": "code", "label": "Mechanism", "language": "python", "value": "runs = [LLM(q, temp=0.7) for _ in range(5)]\nanswers = [extract_answer(r) for r in runs]\nfinal = Counter(answers).most_common(1)[0]" }, { "kind": "list", "label": "Applicable", "items": [ "math/logic problems", "classification" ] } ] }, { "number": "14", "title": "Map-Reduce Summarization", "tag": "parallelism", "description": "For a document longer than the context window: split into chunks, summarize each in parallel (map), then combine the summaries into the final one (reduce). Can be done recursively.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "python", "value": "doc → [chunk1, chunk2, ..., chunkN]\nmap: [summarize(c) for c in chunks] // in parallel\nreduce: summarize(join(summaries)) // final" }, { "kind": "list", "label": "Applicable", "items": [ "long documents", "books", "call transcripts" ] } ] }, { "number": "15", "title": "Prompt-level MoE - specialized agents", "tag": "routing", "description": "Router determines the type of request and routes it to a specialized prompt/agent. Each “expert” is an LLM with a narrow system imperative. This is a prompt-level analogue of an architectural MoE.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "text", "value": "query → Router-LLM: {type: \"code\"|\"math\"|\"write\"}\n ↓\n \"code\" → Code-Expert (system: senior engineer)\n \"math\" → Math-Expert (system: mathematician + CoT)\n \"write\" → Write-Expert (system: copywriter)" }, { "kind": "list", "label": "Applicable", "items": [ "multi-domain products", "reduction of errors by specialization" ] } ] } ] }, { "number": "04", "title": "Chaining & Control Flow", "intro": "", "cards": [ { "number": "16", "title": "Sequential Chain", "tag": "chain", "description": "The output of one LLM call becomes the input of the next. Each step transforms the data: extract → analyze → generate → format. Allows you to break a complex task into manageable subtasks.", "blocks": [ { "kind": "code", "label": "Pipeline example", "language": "text", "value": "raw_text\n → [Step 1] extract_facts(text) → facts_json\n → [Step 2] analyze_sentiment(facts) → sentiment\n → [Step 3] generate_report(facts, sentiment)" }, { "kind": "list", "label": "Applicable", "items": [ "document processing", "content pipelines" ] } ] }, { "number": "17", "title": "Conditional Routing / Branching", "tag": "control flow", "description": "LLM or deterministic code decides on the next step. If condition A is branch 1, if B is branch 2. Allows you to build non-linear pipelines with if-else logic.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "text", "value": "query → classifier → {intent}\n intent == \"complaint\" → escalation_flow\n intent == \"question\" → faq_flow\n intent == \"purchase\" → sales_flow" }, { "kind": "list", "label": "Applicable", "items": [ "chatbots", "ticket systems", "workflow automation" ] } ] }, { "number": "18", "title": "Plan-and-Execute", "tag": "decomposition", "description": "Split into two independent LLM calls: Planner creates a list of steps, Executor executes each step in turn. Planner sees the entire task strategically, Executor is focused on one step - this reduces errors.", "blocks": [ { "kind": "code", "label": "Separation of roles", "language": "python", "value": "Planner(task) → [\"step1\", \"step2\", \"step3\"]\n ↓\nfor step in plan:\n result = Executor(step, context=prev_results)\n context.append(result)" }, { "kind": "list", "label": "Applicable", "items": [ "complex agency tasks", "research", "code generation" ] } ] }, { "number": "19", "title": "Least-to-Most Prompting", "tag": "decomposition", "description": "LLM first breaks down a problem into subtasks from simple to complex. Then solves them sequentially, using simple answers to solve complex ones. An analogue of dynamic programming for LLM.", "blocks": [ { "kind": "code", "label": "Mechanism", "language": "text", "value": "Q: \"How many minutes are in 3.5 days?\"\n→ Sub-Q1: “How many hours are there in a day?” → 24\n→ Sub-Q2: “How many minutes are there in an hour?” → 60\n→ Sub-Q3: “How many minutes are there in 3.5 * 24h?” → 5040" }, { "kind": "list", "label": "Applicable", "items": [ "mathematics", "multi-stage calculations" ] } ] }, { "number": "20", "title": "Iterative Refinement / Self-Edit Loop", "tag": "iteration", "description": "The model generates a draft, criticizes it, improves it - and so on for N iterations. This simulates the editing process. Stopping based on quality condition or number of iterations.", "blocks": [ { "kind": "code", "label": "Cycle", "language": "python", "value": "draft = LLM(task)\nfor i in range(max_iter):\n critique = LLM(f\"Find the flaws: {draft}\")\n if \"no comments\" in critique: break\n draft = LLM(f\"Improve taking into account: {critique}\\n{draft}\")\nreturn draft" }, { "kind": "list", "label": "Applicable", "items": [ "copywriting", "code", "reports", "email drafts" ] } ] } ] }, { "number": "05", "title": "Validation, Retry & Quality Control", "intro": "", "cards": [ { "number": "21", "title": "Schema Validation + Structured Output", "tag": "validation", "description": "Force JSON/structured output via Pydantic, Instructor, function calling or JSON mode. Scheme error → immediate retry with a description of the error in the prompt.", "blocks": [ { "kind": "code", "label": "Pattern", "language": "python", "value": "class Output(BaseModel):\n sentiment: Literal[\"pos\",\"neg\",\"neu\"]\n score: float = Field(ge=0, le=1)\n\nresult = instructor.chat(model, Output, prompt)\n// if parsing error occurs - auto-retry with traceback" }, { "kind": "list", "label": "Applicable", "items": [ "any downstream code", "Integration API", "databases" ] } ] }, { "number": "22", "title": "LLM-as-Judge", "tag": "evaluation", "description": "Separate LLM call to evaluate the quality of the main output. Judge receives the original question + answer and gives a score based on the criteria. Works as an automated reviewer.", "blocks": [ { "kind": "code", "label": "Prompt Judge models", "language": "text", "value": "System: \"You are a strict evaluator of the quality of answers\"\nUser: f\"\"\"\nQuestion: {question}\nAnswer: {answer}\nRate: accuracy (0-5), recall (0-5),\n hallucinations (yes/no)\nReply JSON.\n\"\"\"" }, { "kind": "list", "label": "Applicable", "items": [ "CI/CD for prompts", "online quality control", "RLHF-like feedback" ] } ] }, { "number": "23", "title": "Retry with Error Feedback", "tag": "retry", "description": "If there is an error, do not just repeat the call, but pass on to the next attempt a description of what went wrong. LLM sees his mistake and takes it into account. Exponential backoff for rate limits.", "blocks": [ { "kind": "code", "label": "Smart Retry", "language": "python", "value": "for attempt in range(max_retries):\n result = LLM(prompt + error_context)\n ok, error = validate(result)\n if ok: return result\n error_context += f\"\\n{attempt} failed: {error}\"\n sleep(2 ** attempt) // exponential backoff\nraise MaxRetriesError" }, { "kind": "list", "label": "Applicable", "items": [ "production reliability", "code generation", "parsing" ] } ] }, { "number": "24", "title": "Grounding / Hallucination Detection", "tag": "validation", "description": "Checking that every factual statement in the answer is supported by sources. Either through the NLI model (entailment), or through a separate LLM call, or through a citation search.", "blocks": [ { "kind": "code", "label": "NLI approach", "language": "python", "value": "claims = extract_claims(llm_answer)\nfor claim in claims:\n score = nli_model(premise=source_docs,\n hypothesis=claim)\n // entail / neutral / contradict\n if score == \"contradict\": flag(claim)" }, { "kind": "list", "label": "Applicable", "items": [ "medicine", "jurisprudence", "finance", "RAG QA" ] } ] }, { "number": "25", "title": "Fallback Chain - model degradation", "tag": "resilience", "description": "If the main model fails, automatic transition to the backup model occurs. Hierarchy: first the expensive model, in case of timeout/error - cheaper, then deterministic fallback.", "blocks": [ { "kind": "code", "label": "Chain", "language": "text", "value": "try: return cloud_opus(prompt)\nexcept:\n try: return gpt4o(prompt)\n except:\n try: return cloud_haiku(prompt)\n except: return \"The service is temporarily unavailable\"" }, { "kind": "list", "label": "Applicable", "items": [ "production SLA", "multi-vendor strategy" ] } ] }, { "number": "26", "title": "Guardrails - input/output filters", "tag": "safety", "description": "Security layer before and after the LLM call. Input guardrails: toxicity, PII detection, jailbreak detection. Output guardrails: checking for admissibility, filtering personal data, subject restrictions.", "blocks": [ { "kind": "code", "label": "Two-way protection", "language": "text", "value": "user_input\n → [input_guard] → moderation, PII-masking\n → LLM(safe_input)\n → [output_guard] → toxic filter, PII-unmask\n → user" }, { "kind": "list", "label": "Applicable", "items": [ "B2C products", "GDPR compliance", "corporate chatbots" ] } ] } ] }, { "number": "06", "title": "Retrieval Patterns", "intro": "", "cards": [ { "number": "27", "title": "Hybrid Search (BM25 + Vector)", "tag": "retrieval", "description": "A combination of keyword search (BM25/TF-IDF) and semantic (vector) search. BM25 is precise for exact terms and abbreviations, vector for meaning. RRF (Reciprocal Rank Fusion) combines ranks.", "blocks": [ { "kind": "code", "label": "RRF formula", "language": "tex", "value": "score(doc) = Σ 1 / (k + rank_i(doc))\n// k=60 — smoothing constant\n// summarized across all search engines\nbm25_results + vector_results → RRF → unified_top_k" }, { "kind": "list", "label": "Applicable", "items": [ "enterprise search", "technical documentation" ] } ] }, { "number": "28", "title": "Multi-Query Retrieval", "tag": "retrieval", "description": "LLM generates N rephrases of the original query, each searched in the index, the results deduplicated and merged. Covers different semantic aspects of one question.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "python", "value": "query → LLM → [q1, q2, q3, q4] // N options\n[search(q) for q in queries] // in parallel\n→ deduplicate → top-k → LLM" }, { "kind": "list", "label": "Applicable", "items": [ "Low recall RAG", "multidimensional issues" ] } ] }, { "number": "29", "title": "Iterative / Adaptive Retrieval", "tag": "retrieval", "description": "After the initial response, the model determines what knowledge is missing, formulates a follow-up query and performs an additional search. The cycle continues until the context is complete. This is a RAG with an internal feedback loop.", "blocks": [ { "kind": "code", "label": "Cycle", "language": "python", "value": "context = []\nwhile not sufficient(context):\n gaps = LLM(f\"What is missing for the answer? {context}\")\n new_docs = search(gaps)\n context.extend(new_docs)\nreturn LLM(query, context)" }, { "kind": "list", "label": "Applicable", "items": [ "deep research", "analytical reports" ] } ] }, { "number": "30", "title": "Parent Document Retrieval", "tag": "retrieval", "description": "Index small chunks for precise search, but pass their parent (large) chunks into the context. Search accuracy + context completeness. Different granularities for different tasks.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "text", "value": "Index: small_chunks (128 tokens) → embeddings\nStorage: parent_chunks (512 tokens)\n\nquery → search(small_chunks) → top-k small\n → fetch parent(small_chunk) → into LLM context" }, { "kind": "list", "label": "Applicable", "items": [ "legal documents", "technical manuals" ] } ] } ] }, { "number": "07", "title": "Memory & State", "intro": "", "cards": [ { "number": "31", "title": "Summarization Memory", "tag": "memory", "description": "When the dialogue history exceeds the context window, compress old messages into a compact summary, leaving new ones complete. Rolling summary is updated with each new message.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "python", "value": "if len(history) > threshold:\n old = history[:-10]\n summary = LLM(f\"Summary: {old}\")\n history = [summary_msg(summary)] + history[-10:]" }, { "kind": "list", "label": "Applicable", "items": [ "long dialogues", "assistants" ] } ] }, { "number": "32", "title": "Entity Memory", "tag": "memory", "description": "Extracting and accumulating information about entities (people, companies, products) from the dialogue into a separate store. The next time an entity is mentioned, its profile is pulled into the context.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "text", "value": "entity_store = {}\nmsg → LLM extract → {Ivan: “CTO”, “loves Python”}\nentity_store[\"Ivan\"].update(...)\n\n// with a new message:\ncontext += entity_store.get(detected_entities)" }, { "kind": "list", "label": "Applicable", "items": [ "CRM bots", "personalization", "sales assistants" ] } ] }, { "number": "33", "title": "Episodic Memory (Vector Store)", "tag": "memory", "description": "Past interactions are saved as embeddings. With a new request, search for semantically similar episodes from the past and add them to the context. Long-term memory without storing a full log.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "text", "value": "after each dialogue:\n store(embed(summary), metadata)\n\non a new request:\n memories = search(embed(query), top_k=3)\n context = format_memories(memories) + query" }, { "kind": "list", "label": "Applicable", "items": [ "personal assistants", "Mem0", "MemGPT" ] } ] }, { "number": "34", "title": "Scratchpad / Working Memory", "tag": "memory", "description": "Allocation of a special zone in the prompt for intermediate calculations, notes and agent state. The model writes to the scratchpad during reasoning and reads from it - this is external “working memory”.", "blocks": [ { "kind": "code", "label": "Prompt structure", "language": "text", "value": "[SCRATCHPAD]\nFacts: X=42, Y=18\nPrevious step: found document #3\nCurrent goal: check date\n[/SCRATCHPAD]\n\n[TASK] next step..." }, { "kind": "list", "label": "Applicable", "items": [ "agent systems", "multi-step reasoning" ] } ] } ] }, { "number": "08", "title": "Agentic Patterns", "intro": "", "cards": [ { "number": "35", "title": "Tool Use / Function Calling", "tag": "agentic", "description": "LLM decides when and which tool to call, generates call parameters, receives the result and continues reasoning. Tools: search, calculator, database, API, browser, interpreter code.", "blocks": [ { "kind": "code", "label": "tool-use loop", "language": "text", "value": "LLM → tool_call: {name: \"search\", args: {q: \"...\"}}\nSystem → tool_result: \"...\"\nLLM → tool_call: {name: \"calculator\", args: {...}}\nSystem → tool_result: 42\nLLM → final_answer: \"Answer: 42\"" }, { "kind": "list", "label": "Applicable", "items": [ "automation", "data analysis", "code execution" ] } ] }, { "number": "36", "title": "Reflection - agent self-criticism", "tag": "agentic", "description": "After performing an action, the agent reflects: what happened, what went wrong, what needs to be corrected. This is a supervised self-improvement within one inference loop without updating the weights.", "blocks": [ { "kind": "code", "label": "Cycle", "language": "text", "value": "action = Agent.act(state)\nresult = env.step(action)\nreflection = Agent.reflect(action, result)\n// \"I used the wrong tool because...\"\nnext_action = Agent.act(state, reflection)" }, { "kind": "list", "label": "Applicable", "items": [ "coding agents", "Reflexion (paper)", "debugging" ] } ] }, { "number": "37", "title": "Multi-Agent Orchestration", "tag": "agentic", "description": "Several LLM agents with different roles interact through a common message bus or hierarchically. Orchestrator delegates tasks to specialized subagents and collects the results. Allows you to solve problems that require parallel expertise in different domains.", "blocks": [ { "kind": "code", "label": "Diagram", "language": "text", "value": "Orchestrator\n ├── delegate(\"market analysis\") ──→ Research Agent → report\n ├── delegate(\"financial model\") ──→ Finance Agent → spreadsheet\n └── delegate(\"presentation\") ──→ Writer Agent → slides\n ↓ fan-in ↓\n Orchestrator(report, spreadsheet, slides) → final" }, { "kind": "list", "label": "Applicable", "items": [ "complex business processes", "AutoGen", "CrewAI", "parallel tasks with different expertise" ] } ] }, { "number": "38", "title": "Critic Agent (Constitutional AI-style)", "tag": "agentic", "description": "A dedicated critical agent looks at the output of the main agent and issues specific comments based on the specified criteria (checklist). The main agent fixes it. The critic can be stronger than the main one - asymmetry is useful.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "text", "value": "draft = Worker(task)\ncritique = Critic(draft, criteria=[\n \"accuracy of facts\", \"structure\", \"style\"\n])\nfinal = Worker(task, critique=critique)" }, { "kind": "list", "label": "Applicable", "items": [ "quality of content", "code review", "safety" ] } ] }, { "number": "39", "title": "Tree of Thoughts (ToT)", "tag": "reasoning", "description": "CoT extension: instead of one chain, there is a tree of reasoning options. At each step, N continuations are generated, they are evaluated, and beam search selects the best. Expensive, but powerful for complex planning.", "blocks": [ { "kind": "code", "label": "BFS-ToT scheme", "language": "python", "value": "thoughts = [initial_thought]\nfor depth in range(max_depth):\n candidates = []\n for t in thoughts:\n candidates += generate_k_continuations(t, k=3)\n thoughts = evaluate_and_prune(candidates, keep=beam_w)" }, { "kind": "list", "label": "Applicable", "items": [ "game playing", "creative writing", "complex planning" ] } ] } ] }, { "number": "09", "title": "Output Processing", "intro": "", "cards": [ { "number": "40", "title": "Constrained Decoding / Guided Generation", "tag": "output", "description": "Forced limitation of token space during generation at the logits level. Outlines/LMQL/Guidance masks invalid tokens at every moment of time - 100% guarantee of valid JSON/regex without retry.", "blocks": [ { "kind": "code", "label": "Mechanism", "language": "text", "value": "schema = {\"type\": \"object\", \"properties\": {...}}\n// at each generation step:\nvalid_tokens = grammar.get_valid_next_tokens(current_state)\nlogits[~valid_tokens] = -inf // masking\nnext_token = sample(softmax(logits))" }, { "kind": "list", "label": "Applicable", "items": [ "self-hosted models", "critical structured conclusions" ] } ] }, { "number": "41", "title": "Output Post-Processing Pipeline", "tag": "output", "description": "After receiving the LLM output, a chain of deterministic transformations: strip markdown, extract JSON, normalize, deduplicate, sort, format. The less logic you put on LLM, the more predictable the system.", "blocks": [ { "kind": "code", "label": "Example", "language": "text", "value": "raw = llm_response\n→ strip_think_tags(raw) // remove \n→ extract_json(raw) // regex/parser\n→ validate_schema(json) // pydantic\n→ normalize_values(json) // types, case\n→ enrich_from_db(json) // join with data" }, { "kind": "list", "label": "Applicable", "items": [ "production pipelines", "downstream integration" ] } ] }, { "number": "42", "title": "Streaming + Incremental Processing", "tag": "output", "description": "Processing tokens as they are generated - without waiting for a complete response. Allows you to terminate generation early (early stopping), show progress to the user, and process the beginning of the response in parallel.", "blocks": [ { "kind": "code", "label": "Pattern", "language": "python", "value": "for token in llm.stream(prompt):\n buffer += token\n if detect_complete_sentence(buffer):\n yield process_sentence(buffer)\n buffer = \"\"\n if should_stop_early(buffer):\n llm.cancel()" }, { "kind": "list", "label": "Applicable", "items": [ "UX", "cost optimization", "real-time apps" ] } ] } ] }, { "number": "10", "title": "Optimization & Cost", "intro": "", "cards": [ { "number": "43", "title": "Prompt Compression (LLMLingua)", "tag": "optimization", "description": "Compress long context before sending to expensive model. The small model evaluates the “importance” of each token and removes the unimportant ones. 4-10x compression with minimal quality loss.", "blocks": [ { "kind": "code", "label": "Mechanism", "language": "text", "value": "small_lm.score_tokens(long_context)\n→ perplexity per token\n→ drop low-perplexity tokens (predictable/noisy)\n→ compressed_context (10-25% of the original)\n→ expensive_llm(compressed_context)" }, { "kind": "list", "label": "Applicable", "items": [ "cost reduction", "long documents → expensive models" ] } ] }, { "number": "44", "title": "Semantic Caching", "tag": "optimization", "description": "Caching responses not by exact string match, but by semantic proximity of the request. If cosine distance < threshold - return cache. Reduces costs on repeated requests in different formulations.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "python", "value": "q_emb = embed(query)\ncached = search_cache(q_emb, threshold=0.95)\nif cached: return cached.response // hit\n\nresponse = LLM(query) // miss\ncache.store(q_emb, response)\nreturn response" }, { "kind": "list", "label": "Applicable", "items": [ "FAQ bots", "high-load products" ] } ] }, { "number": "45", "title": "Model Routing — Small/Large", "tag": "optimization", "description": "The query complexity classifier directs simple queries to the cheap, fast model, and complex queries to the expensive one. 70-80% of queries in an average product are simple. RouteLLM, LLM Router.", "blocks": [ { "kind": "code", "label": "Scheme", "language": "python", "value": "complexity = classifier(query) // fast, cheap\nif complexity < 0.4:\n return haiku(query) // $0.25/M tokens\nelif complexity < 0.8:\n return sonnet(query) // $3/M tokens\nelse:\n return opus(query) // $15/M tokens" }, { "kind": "list", "label": "Applicable", "items": [ "cost reduction up to 5x", "latency optimization" ] } ] }, { "number": "46", "title": "Speculative Execution (Draft Model)", "tag": "latency", "description": "The small draft model generates K tokens quickly, the large model verifies them in one forward pass. Accepted tokens are free, rejected tokens are regenerated. 2-3x speedup with the same accuracy.", "blocks": [ { "kind": "code", "label": "Mechanism", "language": "text", "value": "draft_tokens = small_model.generate(k=5) // fast\naccepted = large_model.verify(draft_tokens) // 1 forward\n// accept matches, regenerate the first non-match" }, { "kind": "list", "label": "Applicable", "items": [ "self-hosted models", "latency-sensitive products" ] } ] }, { "number": "47", "title": "Prompt Caching (KV Cache Reuse)", "tag": "optimization", "description": "Anthropic and OpenAI support caching of KV cache prefixes. If system prompt + context are the same between requests, they are not recalculated. Up to 90% savings on prefill tokens for long system prompts.", "blocks": [ { "kind": "code", "label": "Usage pattern", "language": "text", "value": "Structure the prompt: static → up, dynamic → down\n\n[system + docs + examples] ← cached\n────────────────────────────\n[user query] ← changes every time\n\n→ Anthropic: cache_control: {\"type\": \"ephemeral\"}" }, { "kind": "list", "label": "Applicable", "items": [ "long system prompts", "RAG with fixed context" ] } ] } ] } ] }