{ "schemaVersion": "1.0.0", "id": "https://aisdlc.ai/agentic-engineering/learn/catalog.json", "canonicalUrl": "https://aisdlc.ai/agentic-engineering/learn", "title": "Agentic Engineering Field Curriculum", "description": "A source-disciplined learning system of four published paths connecting agentic engineering concepts, practitioner perspectives, implementation patterns, exercises, independent verification, and evidence.", "edition": "2026.08", "reviewedAt": "2026-08-10", "authorityBoundary": "This curriculum is original AISDLC editorial synthesis informed by cited research, official documentation, standards, open-source references, and attributed practitioner perspectives. It is learning material—not a standard, production approval, compliance determination, or proof that a depicted control is implemented.", "completionBoundary": "Progress and completion are self-directed learning records. They are not certification, professional standing, production evidence, or authorization to deploy an agentic system.", "practitionerBoundary": "Practitioner inclusion identifies a relevant independent source. It does not imply contribution, partnership, endorsement, sponsorship, or approval of AISDLC unless a separate public agreement explicitly says so.", "paths": [ { "id": "agentic-engineer-core", "slug": "agentic-engineer-core", "status": "published", "title": "Engineering with agents without losing engineering", "shortTitle": "Agentic engineer core", "promise": "Move from model-assisted coding to a bounded, inspectable, independently challenged delivery system.", "audiences": [ "Engineer", "Architect", "Platform" ], "level": "practitioner", "estimatedHours": 9, "prerequisiteEntryIds": [ "ai-agent", "agentic-engineering" ], "moduleIds": [ "see-the-whole-system", "align-before-implementation", "shape-an-agent-readable-codebase", "slice-for-fast-feedback", "build-against-evidence", "verify-independently", "scale-without-losing-control" ], "capstoneId": "bounded-feature-evidence-dossier", "reviewedAt": "2026-08-10", "canonicalUrl": "https://aisdlc.ai/agentic-engineering/learn/agentic-engineer-core", "markdownUrl": "https://aisdlc.ai/agentic-engineering/learn/agentic-engineer-core/index.md" }, { "id": "private-ai-owned-inference", "slug": "private-ai-owned-inference", "status": "published", "title": "Private AI and owned inference", "shortTitle": "Owned inference", "promise": "Choose deliberately among hosted, dedicated, local, self-hosted, adapted, and trained model paths without treating deployment location as a complete privacy claim.", "audiences": [ "Engineer", "Architect", "Platform", "Risk" ], "level": "advanced", "estimatedHours": 9, "prerequisiteEntryIds": [ "large-language-model", "agentic-engineering", "context-engineering" ], "moduleIds": [ "inference-privacy-boundary", "inference-ownership-levels", "inference-run-local", "inference-serve-and-operate", "inference-adapt-with-evidence", "inference-govern-model-lifecycle" ], "capstoneId": "private-inference-decision-dossier", "reviewedAt": "2026-08-10", "canonicalUrl": "https://aisdlc.ai/agentic-engineering/learn/private-ai-owned-inference", "markdownUrl": "https://aisdlc.ai/agentic-engineering/learn/private-ai-owned-inference/index.md" }, { "id": "agent-platform-engineering", "slug": "agent-platform-engineering", "status": "published", "title": "Agent platform engineering", "shortTitle": "Agent platform", "promise": "Specify the identity, substrate, context, tool, orchestration, verification, evidence, and lifecycle services that bounded agents require, as contracts an independent reviewer can challenge.", "audiences": [ "Architect", "Platform", "Risk", "Operations" ], "level": "advanced", "estimatedHours": 10, "prerequisiteEntryIds": [ "ai-agent", "agentic-engineering", "harness-engineering" ], "moduleIds": [ "platform-execution-substrate", "platform-identity-and-standing", "platform-context-and-memory", "platform-tool-boundaries", "platform-durable-orchestration", "platform-verifier-plane", "platform-evidence-and-lifecycle" ], "capstoneId": "agent-platform-service-dossier", "reviewedAt": "2026-08-10", "canonicalUrl": "https://aisdlc.ai/agentic-engineering/learn/agent-platform-engineering", "markdownUrl": "https://aisdlc.ai/agentic-engineering/learn/agent-platform-engineering/index.md" }, { "id": "lead-ai-native-delivery", "slug": "lead-ai-native-delivery", "status": "published", "title": "Lead AI-native delivery", "shortTitle": "Lead the transition", "promise": "Redesign the operating system around higher generation speed while protecting strategic judgment, independent assurance, and outcome ownership.", "audiences": [ "Executive", "Engineer", "Risk", "Operations" ], "level": "practitioner", "estimatedHours": 9, "prerequisiteEntryIds": [ "agentic-engineering", "organizational-absorption-capacity", "human-accountability" ], "moduleIds": [ "lead-measure-the-whole-system", "lead-find-the-absorption-constraint", "lead-redesign-work-and-attention", "lead-scale-evidence-and-verification", "lead-bind-governance-to-execution", "lead-learn-from-production" ], "capstoneId": "delivery-operating-review", "reviewedAt": "2026-08-10", "canonicalUrl": "https://aisdlc.ai/agentic-engineering/learn/lead-ai-native-delivery", "markdownUrl": "https://aisdlc.ai/agentic-engineering/learn/lead-ai-native-delivery/index.md" } ], "modules": [ { "id": "see-the-whole-system", "index": "01", "title": "See the whole system", "drivingQuestion": "What exists around the model before software can behave as an accountable agentic system?", "capability": "Describe an agent as a bounded system of model, loop, context, tools, harness, substrate, identity, verification, and human authority.", "objectives": [ "Distinguish a model, workflow, agent, harness, and execution substrate.", "Trace where state, tools, permissions, and exit conditions enter an agent loop.", "Identify which layers guide behavior and which layers can enforce a boundary." ], "knowledgeEntryIds": [ "ai-agent", "agentic-engineering", "agent-loop", "harness-engineering", "execution-substrate" ], "insightIds": [ "the-model-is-not-the-architecture" ], "patternIds": [ "bounded-delegation-envelope" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/enterprise-aisdlc" ], "knowledgeSourceIds": [ "anthropic-building-effective-agents", "openai-practical-agent-guide", "openai-harness-engineering" ], "practitionerIds": [ "matt-pocock" ], "lessons": [ { "id": "system-agent-anatomy", "title": "Anatomy of an AI agent", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/ai-agent", "purpose": "Establish the system vocabulary before discussing implementation technique." }, { "id": "system-model-not-architecture", "title": "The model is not the architecture", "format": "read", "minutes": 12, "internalPath": "/insights/the-model-is-not-the-architecture", "purpose": "Separate rented capability from the engineered system around it." }, { "id": "system-practitioner-fundamentals", "title": "Practitioner lens: fundamentals under acceleration", "format": "watch", "minutes": 19, "practitionerSourceId": "matt-pocock-software-fundamentals", "purpose": "Compare the practitioner argument with the AISDLC system and authority boundaries." } ], "exercise": { "brief": "Draw the smallest complete agentic system for one real task and mark every state, action, trust, and decision boundary.", "constraints": [ "Use one bounded task", "Name the human owner", "Separate guidance from enforcement", "Define a finite exit condition" ], "deliverable": "An annotated system anatomy with explicit boundaries and unresolved assumptions.", "verifier": "A reviewer must be able to locate the model, harness, substrate, tools, identity, verifier, evidence sink, and human authority without oral explanation.", "evidence": [ "System anatomy", "Boundary legend", "Assumption register" ] }, "exitCriteria": [ "Can explain why an LLM alone is not an agent", "Can identify where authority is assigned and where constraints are enforced" ] }, { "id": "align-before-implementation", "index": "02", "title": "Align before implementation", "drivingQuestion": "How do people and agents reach enough shared understanding to build the right thing?", "capability": "Turn an ambiguous request into durable intent, explicit decisions, acceptance signals, and a bounded specification without treating the specification as a substitute for code comprehension.", "objectives": [ "Separate facts an agent can research from decisions an accountable person must make.", "Choose conversation, research, or prototype according to the fidelity needed to resolve a question.", "Write intent that carries measurable acceptance, constraints, ownership, and known unknowns." ], "knowledgeEntryIds": [ "intent-engineering", "context-engineering", "working-state", "goal-exit-condition" ], "insightIds": [ "context-is-a-runtime-system" ], "patternIds": [ "context-manifest" ], "visualIds": [ "evidence-decision-spine" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/adoption-workbench" ], "knowledgeSourceIds": [ "github-spec-driven", "github-spec-persistence", "anthropic-context" ], "practitionerIds": [ "matt-pocock" ], "lessons": [ { "id": "align-intent", "title": "Intent as an engineering artifact", "format": "read", "minutes": 14, "internalPath": "/agentic-engineering/intent-engineering", "purpose": "Define what durable intent must carry into execution and review." }, { "id": "align-workflow", "title": "Practitioner lens: idea to implementation workflow", "format": "watch", "minutes": 18, "practitionerSourceId": "matt-pocock-skills-workflow", "purpose": "Study one practitioner flow while keeping its claims and enterprise extensions explicit." }, { "id": "align-spec-models", "title": "Compare specification roles", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/intent-engineering", "purpose": "Distinguish spec-first, spec-anchored, spec-as-source, and naive one-shot spec-to-code behavior." } ], "exercise": { "brief": "Interrogate one feature request until its decisions, unknowns, acceptance signals, and authority are explicit.", "constraints": [ "Ask one decision question at a time", "Research external facts separately", "Prototype visual or behavioral uncertainty", "Do not begin production implementation" ], "deliverable": "An intent brief with decision log, acceptance contract, unresolved questions, and named owner.", "verifier": "A separate reviewer challenges whether every acceptance statement is observable and every consequential decision has a human owner.", "evidence": [ "Intent brief", "Decision log", "Acceptance contract", "Open-question register" ] }, "exitCriteria": [ "Can explain why specification is necessary but insufficient", "Can route facts, decisions, and prototype questions to different resolution methods" ] }, { "id": "shape-an-agent-readable-codebase", "index": "03", "title": "Shape an agent-readable codebase", "drivingQuestion": "What architecture lets humans retain strategic understanding while agents execute more tactical work?", "capability": "Design stable seams, compact interfaces, local feedback, and durable repository context that both humans and agents can navigate.", "objectives": [ "Identify shallow dependency webs that increase context and test cost.", "Design substantial behavior behind a small, explicit interface.", "Preserve a human-readable module map and source-of-truth boundaries." ], "knowledgeEntryIds": [ "harness-engineering", "working-state", "durable-memory", "instruction-data-trust-boundary" ], "insightIds": [ "harness-engineering-is-platform-engineering" ], "patternIds": [ "context-manifest" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/policy-pack" ], "knowledgeSourceIds": [ "openai-harness-engineering", "anthropic-long-running-harness", "anthropic-context" ], "practitionerIds": [ "matt-pocock" ], "lessons": [ { "id": "shape-harness", "title": "Harness engineering as platform engineering", "format": "read", "minutes": 15, "internalPath": "/insights/harness-engineering-is-platform-engineering", "purpose": "Connect codebase shape to the tools and feedback surfaces around agent work." }, { "id": "shape-deep-modules", "title": "Practitioner lens: deliberate module depth", "format": "watch", "minutes": 19, "practitionerSourceId": "matt-pocock-software-fundamentals", "purpose": "Study the deep-module interpretation without copying the source diagram or treating it as a universal architecture." }, { "id": "shape-module-map", "title": "Map interfaces before implementation", "format": "practice", "minutes": 30, "internalPath": "/agentic-engineering/harness-engineering", "purpose": "Make architectural strategy inspectable before tactical generation begins." } ], "exercise": { "brief": "Choose one dependency-heavy area and redesign its boundary on paper before changing code.", "constraints": [ "Preserve observable behavior", "Name the proposed interface", "Show dependent callers", "Define the fastest meaningful feedback command" ], "deliverable": "A before-and-after module map with interface contract and test seam.", "verifier": "A reviewer must be able to explain the new boundary and its failure containment without reading every implementation file.", "evidence": [ "Dependency map", "Interface contract", "Test-seam proposal", "Tradeoff note" ] }, "exitCriteria": [ "Can distinguish small files from deep modules", "Can explain how architecture affects context cost, testing, and human comprehension" ] }, { "id": "slice-for-fast-feedback", "index": "04", "title": "Slice for fast feedback", "drivingQuestion": "How should work be divided so each agent session produces an inspectable end-to-end result?", "capability": "Turn a durable destination into vertically sliced, dependency-aware work that fits bounded contexts and exposes feedback early.", "objectives": [ "Distinguish decision work from implementation work.", "Prefer end-to-end tracer slices over horizontal layer batches.", "Use dependencies, work-in-process limits, and phase boundaries to control concurrency." ], "knowledgeEntryIds": [ "goal-exit-condition", "working-state", "durable-execution", "organizational-absorption-capacity" ], "insightIds": [ "own-the-speed" ], "patternIds": [ "durable-run-ledger" ], "visualIds": [ "evidence-decision-spine" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/adoption-workbench" ], "knowledgeSourceIds": [ "dora-small-batches", "dora-wip-limits", "anthropic-long-running-harness" ], "practitionerIds": [ "matt-pocock" ], "lessons": [ { "id": "slice-wayfinder", "title": "Practitioner lens: map the fog before building", "format": "watch", "minutes": 16, "practitionerSourceId": "matt-pocock-wayfinder", "purpose": "Separate decision mapping from downstream specifications and implementation tickets." }, { "id": "slice-absorption", "title": "Own the speed", "format": "read", "minutes": 14, "internalPath": "/insights/own-the-speed", "purpose": "Connect local generation speed to review, integration, verification, and decision capacity." }, { "id": "slice-tracer-plan", "title": "Write one tracer-slice plan", "format": "practice", "minutes": 35, "internalPath": "/enterprise-aisdlc", "purpose": "Create a small end-to-end path with an external acceptance signal." } ], "exercise": { "brief": "Decompose the aligned feature into end-to-end slices with explicit blocking edges and feedback.", "constraints": [ "Each slice changes observable behavior", "Each slice fits one bounded implementation context", "Every dependency is named", "Limit active implementation to verifier capacity" ], "deliverable": "A tracer-slice map with acceptance and blocking relationships.", "verifier": "A reviewer rejects any slice that is only a database, API, UI, or documentation layer without an independently observable outcome.", "evidence": [ "Slice map", "Blocking graph", "Acceptance path", "Work-in-process rule" ] }, "exitCriteria": [ "Can separate decision tickets from implementation tickets", "Can create a smallest end-to-end slice with a useful feedback signal" ] }, { "id": "build-against-evidence", "index": "05", "title": "Build against evidence", "drivingQuestion": "What feedback must exist before an agent begins changing the system?", "capability": "Design deterministic tests, evaluations, typed outputs, and outcome checks that let construction advance in small verified increments.", "objectives": [ "Translate acceptance statements into externally observable checks.", "Use red-green-refactor and fast feedback where the task permits it.", "Separate implementation self-checks from release-grade independent verification." ], "knowledgeEntryIds": [ "eval-driven-development", "structured-output", "llm-as-judge", "observability" ], "insightIds": [ "verification-is-a-plane-not-a-phase" ], "patternIds": [ "generator-verifier-separation", "evidence-first-delivery" ], "visualIds": [ "evidence-decision-spine" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/assurance-case" ], "knowledgeSourceIds": [ "anthropic-evals", "nist-benchmark-evals", "llm-judge-position-bias" ], "practitionerIds": [ "matt-pocock" ], "lessons": [ { "id": "build-eval", "title": "Eval-driven development", "format": "read", "minutes": 16, "internalPath": "/agentic-engineering/eval-driven-development", "purpose": "Place acceptance and failure hypotheses before implementation." }, { "id": "build-feedback", "title": "Practitioner lens: tests and fast feedback", "format": "watch", "minutes": 19, "practitionerSourceId": "matt-pocock-software-fundamentals", "purpose": "Assess test-driven practice as a construction loop rather than a complete assurance claim." }, { "id": "build-oracles", "title": "Compare deterministic and model-based checks", "format": "compare", "minutes": 24, "internalPath": "/agentic-engineering/llm-as-judge", "purpose": "Choose an oracle according to the failure hypothesis and its own known limitations." } ], "exercise": { "brief": "Write the acceptance and regression checks for one tracer slice before producing its implementation.", "constraints": [ "Include at least one deterministic oracle", "Name nondeterministic tolerances", "Record exact test and evaluator versions", "Do not let the builder define acceptance after seeing the output" ], "deliverable": "A runnable verification contract and empty evidence manifest.", "verifier": "An independent reviewer challenges whether the checks would detect a plausible wrong implementation, not merely confirm execution.", "evidence": [ "Failing baseline", "Verification contract", "Evaluator manifest", "Expected evidence schema" ] }, "exitCriteria": [ "Can explain why passing self-authored checks is not independent assurance", "Can match deterministic and model-based checks to distinct failure hypotheses" ] }, { "id": "verify-independently", "index": "06", "title": "Verify independently", "drivingQuestion": "Who or what can credibly challenge the builder before a consequential result advances?", "capability": "Design an independent verifier with named ownership, trigger, scope, block authority, method, and recorded disposition.", "objectives": [ "Distinguish fresh-context self-review from zero-trust independent verification.", "Combine algorithmic and agentic checks rather than choosing one exclusively.", "Escalate disagreement to named human authority instead of averaging it away." ], "knowledgeEntryIds": [ "multi-agent-verification", "human-accountability", "evidence-engineering", "deterministic-containment" ], "insightIds": [ "verification-is-a-plane-not-a-phase", "evidence-before-autonomy" ], "patternIds": [ "generator-verifier-separation", "evidence-first-delivery" ], "visualIds": [ "governed-agentic-stack", "evidence-decision-spine" ], "blueprintIds": [ "transactional-agent-with-approval" ], "toolPaths": [ "/assurance-case", "/policies" ], "knowledgeSourceIds": [ "nist-zero-trust", "anthropic-evals", "nist-benchmark-evals" ], "practitionerIds": [ "matt-pocock" ], "lessons": [ { "id": "verify-plane", "title": "Verification is a plane, not a phase", "format": "read", "minutes": 15, "internalPath": "/insights/verification-is-a-plane-not-a-phase", "purpose": "Apply challenge across intent, build, release, and operation." }, { "id": "verify-fresh-context", "title": "Fresh context versus independent challenge", "format": "compare", "minutes": 18, "practitionerSourceId": "matt-pocock-software-fundamentals", "purpose": "Preserve the value of a fresh review while locating the stronger enterprise independence boundary." }, { "id": "verify-design", "title": "Design a blocking verifier", "format": "practice", "minutes": 35, "internalPath": "/assurance-case", "purpose": "Make ownership, block semantics, and evidence inspectable." } ], "exercise": { "brief": "Specify one named verifier for the tracer slice and run it against a deliberately flawed candidate.", "constraints": [ "The builder cannot own final disposition", "Use a different model lineage or deterministic method where applicable", "Define block and escalation behavior", "Record disagreement without erasing it" ], "deliverable": "A verifier contract, challenged candidate, finding, and disposition record.", "verifier": "A human assurance owner confirms that the check has a defined trigger, scope, authority to block, and recorded disposition.", "evidence": [ "Verifier identity and version", "Challenge input", "Finding", "Block or pass decision", "Disposition" ] }, "exitCriteria": [ "Can state why no agent verifies its own output", "Can design fused algorithmic and agentic verification with a named escalation path" ] }, { "id": "scale-without-losing-control", "index": "07", "title": "Scale without losing control", "drivingQuestion": "How does an organization absorb more agent output without surrendering quality, accountability, or comprehension?", "capability": "Balance generation with product decisions, verifier capacity, integration, evidence, runtime intervention, and named human release authority.", "objectives": [ "Diagnose the value-stream constraint instead of treating pull-request volume as delivered value.", "Scale verifier and integration capacity with generation capacity.", "Connect release evidence to runtime monitoring, intervention, recertification, and retirement." ], "knowledgeEntryIds": [ "software-factory", "organizational-absorption-capacity", "runtime-policy-enforcement", "agent-incident-response", "continuous-recertification-retirement" ], "insightIds": [ "own-the-speed", "evidence-before-autonomy" ], "patternIds": [ "risk-adaptive-autonomy", "agent-identity-lifecycle", "durable-run-ledger" ], "visualIds": [ "risk-adaptive-autonomy-field", "governed-agentic-stack" ], "blueprintIds": [ "transactional-agent-with-approval" ], "toolPaths": [ "/agent-estate", "/control-plane-lab" ], "knowledgeSourceIds": [ "google-dora-ai-development", "github-agent-pr-review", "openai-monitoring" ], "practitionerIds": [ "matt-pocock" ], "lessons": [ { "id": "scale-absorption", "title": "Organizational absorption capacity", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/organizational-absorption-capacity", "purpose": "Model the rate at which generated work becomes understood, verified, integrated, authorized, and operated value." }, { "id": "scale-software-factory", "title": "Factory or queue amplifier?", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/software-factory", "purpose": "Separate repeatable production systems from uncontrolled artifact generation." }, { "id": "scale-control-rehearsal", "title": "Rehearse a hold, correction, and release", "format": "practice", "minutes": 35, "internalPath": "/control-plane-lab", "purpose": "Practice finite intervention verbs while preserving the simulation boundary." } ], "exercise": { "brief": "Map one delivery value stream and redesign it for higher verified throughput rather than higher artifact volume.", "constraints": [ "Measure queues and rework", "Protect named human decision windows", "Scale verifier ownership before release volume", "Define degradation and stop conditions" ], "deliverable": "A current-state and target-state absorption map with operating limits.", "verifier": "Engineering, assurance, and product owners each identify their queue, decision right, evidence need, and stop condition.", "evidence": [ "Value-stream map", "Capacity assumptions", "Work-in-process limits", "Authority map", "Runtime stop conditions" ] }, "exitCriteria": [ "Can distinguish generation velocity from verified system throughput", "Can describe how evidence and runtime signals reopen lifecycle decisions" ] }, { "id": "inference-privacy-boundary", "index": "01", "title": "Trace where the data comes to rest", "drivingQuestion": "Where does your data come to rest once a prompt leaves the application?", "capability": "Trace every place prompts, retrieved context, memory, tool arguments, telemetry, backups, identities, and operator access carry data across a boundary.", "objectives": [ "Follow one request from user input through the model call, tool calls, traces, logs, and backups.", "Separate the claim that weights run on your hardware from the claim that data stays inside a boundary.", "Name every identity and operator that can read each store the request touches.", "Record confirmed retention periods and mark the rest as unknown." ], "knowledgeEntryIds": [ "private-ai", "context-engineering", "durable-memory", "instruction-data-trust-boundary", "observability", "agent-identity" ], "insightIds": [ "context-is-a-runtime-system" ], "patternIds": [ "context-manifest", "tool-contract-firewall" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "regulated-read-only-agent" ], "toolPaths": [ "/policy-pack" ], "knowledgeSourceIds": [ "owasp-memory", "anthropic-context", "opentelemetry-genai", "nist-zero-trust" ], "practitionerIds": [], "lessons": [ { "id": "inference-privacy-what-private-means", "title": "What private AI claims and what it does not", "format": "read", "minutes": 16, "internalPath": "/agentic-engineering/private-ai", "purpose": "Separate where weights run from where data goes." }, { "id": "inference-privacy-store-comparison", "title": "Compare the stores a single request creates", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/durable-memory", "purpose": "Set memory, traces, and backups side by side before mapping." }, { "id": "inference-privacy-trace-request", "title": "Trace one request end to end", "format": "practice", "minutes": 30, "internalPath": "/agentic-engineering/context-engineering", "purpose": "Produce the data-path map later decisions are argued against." } ], "exercise": { "brief": "Trace one workload you already run and record every place its data comes to rest or crosses an operator boundary.", "constraints": [ "Use one workload you already operate, not a hypothetical one.", "Follow the request past the model call into traces, logs, memory, and backups.", "Name every identity and operator with read access to each store.", "Record retention you can confirm in configuration and mark the rest unknown.", "Do not record a boundary you cannot show in a configuration file or console." ], "deliverable": "A data-path map for one workload with a store-by-store access and retention register.", "verifier": "A reviewer who did not build the map picks two stores at random and asks the learner to show the configuration that proves the recorded access and retention.", "evidence": [ "Data-path map", "Store register", "Identity and operator access list", "Retention notes separating confirmed from unknown", "Unresolved question list" ] }, "exitCriteria": [ "Can explain why running weights locally does not by itself bound where data travels.", "Can name the stores in their own workload that hold prompt or retrieved context data." ] }, { "id": "inference-ownership-levels", "index": "02", "title": "Choose how much of the stack to own", "drivingQuestion": "How much of the inference stack do you need to own to satisfy the constraint you actually have?", "capability": "Compare hosted APIs, dedicated endpoints, self-hosted open weights, adapted models, and pretraining against a written constraint rather than a preference.", "objectives": [ "State the binding constraint as residency, contractual control, latency, cost, or capability before comparing options.", "Compare the ownership levels on what each one changes and what each one leaves unchanged.", "Record the operating burden each level transfers onto your own team.", "Name the alternative you rejected and the evidence that rejected it.", "State the conditions under which pretraining or continued pretraining would be considered, and why they do not apply to this workload." ], "knowledgeEntryIds": [ "private-ai", "open-weight-model", "self-hosted-inference", "model-adaptation", "large-language-model", "model-routing" ], "insightIds": [ "the-model-is-not-the-architecture" ], "patternIds": [ "risk-adaptive-autonomy" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "regulated-read-only-agent" ], "toolPaths": [ "/enterprise-aisdlc", "/adoption-workbench" ], "knowledgeSourceIds": [ "openai-gpt-oss", "openai-gpt-oss-model-card", "nist-ai-rmf" ], "practitionerIds": [], "lessons": [ { "id": "inference-ownership-open-weights", "title": "What open weights change", "format": "read", "minutes": 14, "internalPath": "/agentic-engineering/open-weight-model", "purpose": "Establish what possession of weights does and does not grant." }, { "id": "inference-ownership-level-comparison", "title": "Compare the ownership levels side by side", "format": "compare", "minutes": 22, "internalPath": "/agentic-engineering/self-hosted-inference", "purpose": "Force each ownership level to answer the same questions." }, { "id": "inference-ownership-decision-record", "title": "Write the decision against the constraint", "format": "practice", "minutes": 30, "internalPath": "/adoption-workbench", "purpose": "Convert the comparison into a record another person can challenge." } ], "exercise": { "brief": "Write a one-page ownership decision for a real workload and defend it against the adjacent ownership levels in both directions.", "constraints": [ "State the binding constraint in one sentence before comparing anything.", "Give cost, latency, residency, and capability each a measured or explicitly estimated figure.", "Name the team that would carry the operating burden at the chosen level.", "Include the rejected option and the evidence that rejected it.", "Do not select a level you have no staffing plan to operate.", "Record pretraining as considered and excluded with the reason, or as in scope with the data and staffing it would require." ], "deliverable": "An ownership decision record for one workload with the binding constraint, the comparison, and the rejected alternatives.", "verifier": "A reviewer argues for the adjacent level — the cheaper one below the choice, or the higher-ownership one when the lowest level was chosen — and the learner must answer with the binding constraint and the recorded figures rather than with preference.", "evidence": [ "Constraint statement", "Option comparison table", "Operating-burden note", "Rejected-option rationale", "Named decision owner" ] }, "exitCriteria": [ "Can name the single constraint that would change the decision if it moved.", "Can state what each ownership level does not solve." ] }, { "id": "inference-run-local", "index": "03", "title": "Run inference on hardware you control", "drivingQuestion": "What does a model actually need from your hardware before it can carry a real workload?", "capability": "Match model format, quantization, memory, context length, and latency to a workload you measure on hardware you control.", "objectives": [ "Relate parameter count, quantization format, and context length to a memory footprint.", "Measure first-token latency and sustained throughput on your own machine instead of quoting published figures.", "Identify which of your workloads a local model can carry and which it cannot.", "Record the quality difference between quantization levels, or record that you did not measure it." ], "knowledgeEntryIds": [ "local-inference", "quantization", "open-weight-model", "context-window", "inference-serving" ], "insightIds": [ "harness-engineering-is-platform-engineering" ], "patternIds": [ "evidence-first-delivery" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/enterprise-aisdlc" ], "knowledgeSourceIds": [ "llama-cpp-local-inference", "ollama-local-only", "hf-quantization", "artificial-analysis-methodology" ], "practitionerIds": [], "lessons": [ { "id": "inference-local-what-it-costs", "title": "Local inference and what it costs you", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/local-inference", "purpose": "Name the resource limits before touching a model." }, { "id": "inference-local-quantization-comparison", "title": "Compare quantization levels of one model", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/quantization", "purpose": "Separate the memory saving from the quality cost." }, { "id": "inference-local-hardware-measurement", "title": "Measure your own hardware", "format": "practice", "minutes": 35, "internalPath": "/agentic-engineering/local-inference", "purpose": "Replace published benchmarks with figures from the target machine." } ], "exercise": { "brief": "Run one open-weight model on hardware you control and record what it can and cannot carry for a real task of yours. Do this even where the chosen ownership level is hosted — the measurement is what makes that choice defensible rather than assumed.", "constraints": [ "Use your own hardware and your own task, not a published benchmark task.", "Test at least two quantization levels of the same model.", "Record memory headroom at the longest context you intend to use.", "Report first-token latency and sustained throughput as separate figures.", "Record the quality difference you observed, or record explicitly that you did not measure it." ], "deliverable": "A hardware fit report for one model and one task, with measured latency, memory headroom, and quality observations.", "verifier": "A reviewer reruns one recorded configuration on the same machine and checks whether the numbers reproduce within the tolerance the learner stated.", "evidence": [ "Model and quantization identifiers", "Hardware specification", "Latency measurements", "Memory-headroom record", "Quality observation notes" ] }, "exitCriteria": [ "Can estimate whether a given model fits a given machine before downloading it.", "Can name a workload of their own that local inference is not suited to carry." ] }, { "id": "inference-serve-and-operate", "index": "04", "title": "Serve it like production infrastructure", "drivingQuestion": "What breaks once a model you host becomes a dependency other teams rely on?", "capability": "Engineer capacity, isolation, routing, patching, observability, and failure recovery for an inference endpoint you own.", "objectives": [ "Size concurrency and queueing against a stated latency target.", "Isolate the serving workload from the credentials, tenants, and network paths it does not need.", "Define fallback routing and state the failure modes it does not cover.", "Instrument the endpoint so degradation is visible before a user reports it.", "Define who patches the serving runtime, the model artifact, and the host, and how a patch is rolled back." ], "knowledgeEntryIds": [ "inference-serving", "self-hosted-inference", "model-routing", "observability-control", "execution-substrate", "sandbox" ], "insightIds": [ "harness-engineering-is-platform-engineering", "own-the-speed" ], "patternIds": [ "tool-contract-firewall", "durable-run-ledger" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "regulated-read-only-agent" ], "toolPaths": [ "/control-plane-lab", "/agent-estate" ], "knowledgeSourceIds": [ "vllm-serving", "opentelemetry-genai", "google-agent-substrate", "nist-deployed-monitoring" ], "practitionerIds": [], "lessons": [ { "id": "inference-serving-as-dependency", "title": "Inference serving as an operated dependency", "format": "read", "minutes": 16, "internalPath": "/agentic-engineering/inference-serving", "purpose": "Move from a model that runs to a service others depend on." }, { "id": "inference-serving-routing-comparison", "title": "Compare routing and fallback under failure", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/model-routing", "purpose": "Examine each fallback against the failure it must absorb." }, { "id": "inference-serving-failure-rehearsal", "title": "Rehearse a degradation and a recovery", "format": "practice", "minutes": 35, "internalPath": "/control-plane-lab", "purpose": "Rehearse one failure in simulation before production supplies one." } ], "exercise": { "brief": "Write the operating plan for one self-hosted endpoint and rehearse the failure you consider most likely. Treat this as a rehearsal even where the chosen ownership level is hosted.", "constraints": [ "State a latency target and the concurrency at which the endpoint holds it.", "Name the isolation boundary between the serving process and everything it can reach.", "Define what happens when the primary path fails, including the case where the fallback is refusal.", "Rehearse one failure and record the actual recovery time.", "Keep the rehearsal outside production.", "Name the patch owner and the tested rollback path for the runtime and the model artifact." ], "deliverable": "An operating plan for one endpoint plus a written record of one rehearsed failure and its recovery.", "verifier": "An operations reviewer who did not run the rehearsal reads the plan, states which signal should have revealed the degradation, then checks whether that signal exists.", "evidence": [ "Capacity and latency targets", "Isolation description", "Routing and fallback rules", "Failure-rehearsal record", "Recovery time and remaining gaps", "Patch and rollback ownership record" ] }, "exitCriteria": [ "Can state the concurrency at which their endpoint stops meeting its latency target.", "Can name the signal that reveals degradation before a user reports it." ] }, { "id": "inference-adapt-with-evidence", "index": "05", "title": "Adapt only with evidence", "drivingQuestion": "What measured gap justifies changing the weights instead of changing the context?", "capability": "Choose among prompting, retrieval, parameter-efficient tuning, full fine-tuning, and pretraining from a reproduced failure rather than an assumption.", "objectives": [ "Reproduce a failure and classify it as missing knowledge, missing format, missing behavior, or missing capability.", "Choose the first intervention from the failure classification and record the measured result before escalating.", "Hold an evaluation set fixed across every intervention so the comparison stays honest.", "State the recurring cost an adapted model adds, including re-evaluation on every base-model change." ], "knowledgeEntryIds": [ "model-adaptation", "parameter-efficient-fine-tuning", "retrieval-augmented-generation", "eval-driven-development", "context-engineering", "structured-output" ], "insightIds": [ "evidence-before-autonomy", "verification-is-a-plane-not-a-phase" ], "patternIds": [ "evidence-first-delivery", "generator-verifier-separation" ], "visualIds": [ "evidence-decision-spine" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/assurance-case" ], "knowledgeSourceIds": [ "hf-peft-methods", "lora-paper", "hf-fine-tuning", "anthropic-evals" ], "practitionerIds": [], "lessons": [ { "id": "inference-adapt-what-changes", "title": "What adaptation actually changes", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/model-adaptation", "purpose": "Separate the failure classes weights can fix from those they cannot." }, { "id": "inference-adapt-intervention-comparison", "title": "Compare context, retrieval, and tuning on one failure", "format": "compare", "minutes": 22, "internalPath": "/agentic-engineering/parameter-efficient-fine-tuning", "purpose": "Match each intervention to the failure class it can repair." }, { "id": "inference-adapt-evaluation-first", "title": "Build the evaluation before the training run", "format": "practice", "minutes": 35, "internalPath": "/agentic-engineering/eval-driven-development", "purpose": "Freeze the measurement before any intervention runs." } ], "exercise": { "brief": "Take one failing behavior, freeze an evaluation set against it, and record how far the cheapest interventions get before any weight change is considered.", "constraints": [ "Freeze the evaluation set before the first intervention and do not edit it afterwards.", "Choose the first intervention from the failure classification, not from cost order, and record why it is the right class of fix.", "Record the measured result of each intervention before escalating.", "Propose a weight change only where the recorded failure class is one weights can fix and cheaper interventions in that class are recorded as insufficient.", "State the re-evaluation an adapted model would need on every base-model change." ], "deliverable": "An adaptation decision record holding the frozen evaluation set, the success threshold stated before each intervention ran, the measured result of each escalation step, and the stop condition if the final intervention misses.", "verifier": "A reviewer who did not run the interventions reruns the frozen evaluation against the learner's best untuned configuration, confirms the reported gap, and checks the escalation record: a weight change proposed without a recorded insufficient result from a cheaper intervention is returned for rework.", "evidence": [ "Frozen evaluation set", "Failure classification", "Measured result of each escalation step", "Recurring cost and re-evaluation note", "Adaptation recommendation with rejected alternatives" ] }, "exitCriteria": [ "Can name a failure class that tuning does not fix.", "Can state the recurring cost an adapted model adds to every base-model upgrade." ] }, { "id": "inference-govern-model-lifecycle", "index": "06", "title": "Govern the model lifecycle", "drivingQuestion": "Who owns a model once it is running, and what makes it stop being allowed to run?", "capability": "Track license, provenance, evaluation, vulnerability, approval, recertification, and retirement for every model a system calls.", "objectives": [ "Record license terms and provenance for each model, including weights obtained through a redistributor.", "Bind every model version to one accountable person rather than a team alias.", "Define the recertification trigger for a model version and the evidence a recertification requires.", "Define retirement, including what happens to workloads still depending on the retired version." ], "knowledgeEntryIds": [ "agent-estate-governance", "continuous-recertification-retirement", "evidence-engineering", "human-accountability", "risk-tiered-autonomy", "open-weight-model" ], "insightIds": [ "every-agent-is-a-governed-principal", "evidence-before-autonomy" ], "patternIds": [ "agent-identity-lifecycle", "durable-run-ledger" ], "visualIds": [ "evidence-decision-spine", "risk-adaptive-autonomy-field" ], "blueprintIds": [ "transactional-agent-with-approval" ], "toolPaths": [ "/agent-estate", "/policies" ], "knowledgeSourceIds": [ "slsa-provenance", "nist-ai-rmf", "microsoft-agent-lifecycle", "openai-gpt-oss-model-card" ], "practitionerIds": [], "lessons": [ { "id": "inference-govern-recertification", "title": "Recertification and retirement of a model version", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/continuous-recertification-retirement", "purpose": "Treat an approved model as a dated decision." }, { "id": "inference-govern-license-comparison", "title": "Compare license and provenance claims across model sources", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/open-weight-model", "purpose": "Compare license, model card, and redistribution path claims." }, { "id": "inference-govern-model-register", "title": "Build a model register with owners and expiry", "format": "practice", "minutes": 35, "internalPath": "/agent-estate", "purpose": "Make every model in use attributable to a person and a date." } ], "exercise": { "brief": "Build a register of every model your systems currently call and give each row an owner, a license record, a provenance record, and an expiry date.", "constraints": [ "Include hosted endpoints, not only weights you downloaded.", "Record the exact version or snapshot identifier, never the model family name alone.", "Give every row one accountable person rather than a team alias.", "Give every row a recertification date and the event that would pull it forward.", "Mark unverified license or provenance as unverified rather than assuming it.", "Record where advisories for each model, runtime, and adapter are monitored, and who receives them." ], "deliverable": "A model register covering every model in use, with owner, exact version, license, provenance, evaluation reference, vulnerability watch source, and expiry.", "verifier": "A reviewer picks one row and asks the named owner to produce the license text and the provenance record; a row that cannot be evidenced is recorded as unverified.", "evidence": [ "Model register", "License and provenance records", "Evaluation references", "Recertification schedule", "Retirement plan for one superseded version", "Vulnerability monitoring source per row" ] }, "exitCriteria": [ "Can name the accountable person for every model their systems call.", "Can state the event that would trigger recertification before the scheduled date." ] }, { "id": "platform-execution-substrate", "index": "01", "title": "Specify the execution substrate", "drivingQuestion": "Where does agent work actually take effect, and what stops a failed run from reaching anything else?", "capability": "Specify isolated workspaces, scoped credentials, controlled egress, and reproducible recovery for every agent run.", "objectives": [ "Separate the layer that reasons from the layer where actions take effect.", "Specify workspace, filesystem, process, and network isolation for a stated risk tier.", "Inject credentials per run with an expiry so no run holds standing secrets.", "Define snapshot, teardown, and recovery behavior for an interrupted run." ], "knowledgeEntryIds": [ "execution-substrate", "sandbox", "deterministic-containment", "harness-engineering", "risk-tiered-autonomy" ], "insightIds": [ "harness-engineering-is-platform-engineering", "the-model-is-not-the-architecture" ], "patternIds": [ "bounded-delegation-envelope" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/enterprise-aisdlc", "/control-plane-lab" ], "knowledgeSourceIds": [ "google-agent-substrate", "anthropic-managed-agents", "nist-zero-trust", "owasp-agentic-top10" ], "practitionerIds": [], "lessons": [ { "id": "platform-substrate-layer", "title": "The layer where actions take effect", "format": "read", "minutes": 16, "internalPath": "/agentic-engineering/execution-substrate", "purpose": "Fix the boundary between reasoning and consequence first." }, { "id": "platform-substrate-isolation-compare", "title": "Match isolation strength to blast radius", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/sandbox", "purpose": "Choose isolation strength from what a failed run could reach." }, { "id": "platform-substrate-contract-practice", "title": "Write one environment contract", "format": "practice", "minutes": 30, "internalPath": "/control-plane-lab", "purpose": "Turn isolation intent into a written contract inside the lab's simulation boundary." } ], "exercise": { "brief": "Write the substrate contract for one agent workload and show what a failed or hostile run cannot reach.", "constraints": [ "Choose one workload and state its risk tier", "State filesystem, process, and network boundaries separately", "Issue credentials per run with a stated expiry", "Define teardown and recovery for a run interrupted mid-step" ], "deliverable": "A substrate contract carrying isolation boundaries, credential handling, and a recovery procedure.", "verifier": "A platform reviewer who did not draft the contract attempts to name an escape path — a shared filesystem, an ambient credential, or an open egress route — that the written contract fails to close, and records the finding without oral explanation from the author.", "evidence": [ "Substrate contract", "Isolation boundary map", "Credential lifecycle note", "Recovery procedure", "Residual exposure list" ] }, "exitCriteria": [ "Can explain why a prompt-level instruction is guidance and not containment", "Can state what a run can reach at each isolation tier and who pays the cost of the stronger tier" ] }, { "id": "platform-identity-and-standing", "index": "02", "title": "Give every agent standing", "drivingQuestion": "Whose authority does an agent act under, and what happens to that standing when nobody renews it?", "capability": "Design an identity record that binds each agent to a purpose, a sponsor, an accountable owner, scoped permissions, and an expiry that removes standing by default.", "objectives": [ "Separate agent identity from the human account or shared service account it borrows today.", "Bind delegated permission to a stated purpose and a named accountable owner.", "Treat issuance, renewal, suspension, and revocation as one lifecycle rather than an onboarding step.", "Record what an agent may never do regardless of who asks." ], "knowledgeEntryIds": [ "agent-identity", "agent-estate-governance", "human-accountability", "continuous-recertification-retirement", "risk-tiered-autonomy" ], "insightIds": [ "every-agent-is-a-governed-principal" ], "patternIds": [ "agent-identity-lifecycle", "bounded-delegation-envelope" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "regulated-read-only-agent" ], "toolPaths": [ "/agent-estate", "/policies" ], "knowledgeSourceIds": [ "nist-agent-identity", "microsoft-agent-identities", "microsoft-agent-owners-sponsors", "microsoft-agent-id-governance" ], "practitionerIds": [], "lessons": [ { "id": "platform-identity-standing", "title": "Identity and delegated authority", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/agent-identity", "purpose": "Establish standing as a platform service before any permission is attached to it." }, { "id": "platform-identity-estate-read", "title": "From one agent to the estate register", "format": "read", "minutes": 18, "internalPath": "/agentic-engineering/agent-estate-governance", "purpose": "Scale one identity record to a register of every agent and owner." }, { "id": "platform-identity-record-practice", "title": "Draft an identity record with an expiry", "format": "practice", "minutes": 28, "internalPath": "/agent-estate", "purpose": "Make sponsor, owner, purpose, scope, and expiry concrete for one agent." } ], "exercise": { "brief": "Write the identity record for one agent so that its standing lapses unless a named human renews it.", "constraints": [ "Name a sponsor and an accountable owner who are not the person designing the agent", "Scope permissions to the stated purpose and nothing adjacent to it", "Set an expiry date and a default behavior when it passes", "Record the actions this agent may never take regardless of request" ], "deliverable": "An identity record carrying purpose, sponsorship, ownership, permission scope, expiry, and revocation path.", "verifier": "A risk reviewer who did not write the record attempts to find one permission the stated purpose does not justify and one path by which an orphaned agent would keep running after its expiry lapses, and records both findings.", "evidence": [ "Identity record", "Permission scope statement", "Sponsor and owner attestation", "Expiry and renewal rule", "Revocation runbook" ] }, "exitCriteria": [ "Can explain why a shared service account cannot carry agent accountability", "Can describe how standing expires by default instead of persisting after everyone stops watching" ] }, { "id": "platform-context-and-memory", "index": "03", "title": "Version context and memory", "drivingQuestion": "Which text in an agent's window is trusted instruction, and which is untrusted data that arrived from somewhere else?", "capability": "Design context assembly so instructions, retrieval, and working state carry a version, a provenance record, a trust class, and a retention rule.", "objectives": [ "Classify every context source as trusted instruction or untrusted data at assembly time.", "Version instructions and retrieval sets so a past run can be reconstructed later.", "Set retention and expiry for memory that would otherwise carry stale or poisoned content forward.", "Define how a compromised source is detected and removed from future runs." ], "knowledgeEntryIds": [ "context-engineering", "instruction-data-trust-boundary", "durable-memory", "working-state", "retrieval-augmented-generation", "context-hygiene" ], "insightIds": [ "context-is-a-runtime-system" ], "patternIds": [ "context-manifest" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "regulated-read-only-agent" ], "toolPaths": [ "/policy-pack" ], "knowledgeSourceIds": [ "anthropic-context", "owasp-memory", "owasp-agentic-top10" ], "practitionerIds": [], "lessons": [ { "id": "platform-context-runtime", "title": "Context is a runtime system", "format": "read", "minutes": 16, "internalPath": "/insights/context-is-a-runtime-system", "purpose": "Treat the context window as versioned infrastructure, not a prompt." }, { "id": "platform-context-trust-compare", "title": "Instruction and data at the same boundary", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/instruction-data-trust-boundary", "purpose": "Separate what the platform authored from what the agent read." }, { "id": "platform-context-manifest-practice", "title": "Write a context manifest for one run", "format": "practice", "minutes": 30, "internalPath": "/agentic-engineering/context-engineering", "purpose": "Make every source, version, and trust class inspectable after the run." } ], "exercise": { "brief": "Assemble one agent run's context from a written manifest and mark the trust class and retention of every source.", "constraints": [ "Version the system instructions and the retrieval set separately", "Label each source as trusted instruction or untrusted data", "State a retention or expiry rule for anything written to memory", "Record how a poisoned source would be detected and removed" ], "deliverable": "A context manifest with source versions, trust classes, retention rules, and a removal path.", "verifier": "A reviewer reconstructs the same context from the manifest alone and challenges any source whose version, origin, or trust class cannot be established.", "evidence": [ "Context manifest", "Trust classification table", "Instruction version record", "Retention and expiry rule", "Compromised-source removal note" ] }, "exitCriteria": [ "Can explain why retrieved content stays untrusted data even when it comes from an internal system", "Can reconstruct which instructions and sources produced a past run without asking the people who ran it" ] }, { "id": "platform-tool-boundaries", "index": "04", "title": "Draw the tool boundary", "drivingQuestion": "What separates an agent that can read a system, one that can propose a change, and one that can commit it?", "capability": "Design typed tool contracts that separate inspect, propose, execute, and commit capability, each with its own authorization and audit record.", "objectives": [ "Split one broad convenience tool into distinct inspect, propose, execute, and commit capabilities.", "Type inputs and outputs so an out-of-scope call fails at the contract rather than at the system.", "Attach authorization, rate limits, and an audit record to every capability that changes state.", "Record refusals as carefully as successful calls." ], "knowledgeEntryIds": [ "tools-and-protocols", "tool-calling", "model-context-protocol", "structured-output", "runtime-policy-enforcement" ], "insightIds": [ "every-agent-is-a-governed-principal" ], "patternIds": [ "tool-contract-firewall", "bounded-delegation-envelope" ], "visualIds": [ "governed-agentic-stack" ], "blueprintIds": [ "transactional-agent-with-approval" ], "toolPaths": [ "/policy-pack", "/policies" ], "knowledgeSourceIds": [ "mcp-enterprise-auth", "mcp-architecture", "opa-policy-enforcement", "owasp-agentic-top10" ], "practitionerIds": [], "lessons": [ { "id": "platform-tool-contracts", "title": "Where reasoning stops and action starts", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/tools-and-protocols", "purpose": "Establish the contract surface as the last deterministically enforceable boundary." }, { "id": "platform-tool-capability-compare", "title": "Four capability classes behind one endpoint", "format": "compare", "minutes": 18, "internalPath": "/agentic-engineering/tool-calling", "purpose": "Show what changes when inspect, propose, execute, and commit stop sharing a single permission." }, { "id": "platform-tool-typed-practice", "title": "Write a typed tool contract", "format": "practice", "minutes": 30, "internalPath": "/policy-pack", "purpose": "Push scope, authorization, and audit into the interface instead of the instructions." } ], "exercise": { "brief": "Take one broad tool an agent would use and split it into separately authorized capabilities with typed contracts.", "constraints": [ "Define inspect, propose, execute, and commit as distinct capabilities", "Type every input and reject anything outside the declared schema", "Require a different authorization for any capability that changes state", "Emit an audit record for every call, including refusals" ], "deliverable": "A tool contract set with typed schemas, an authorization matrix, and audit fields.", "verifier": "A reviewer proposes an out-of-scope call for each capability and confirms the written contract rejects it without depending on model cooperation.", "evidence": [ "Capability split", "Typed schemas", "Authorization matrix", "Refusal cases", "Audit record fields" ] }, "exitCriteria": [ "Can explain why one write-capable tool collapses four separate risk decisions into one", "Can point to where a refusal happens in the contract rather than in the model" ] }, { "id": "platform-durable-orchestration", "index": "05", "title": "Design durable runs", "drivingQuestion": "What happens to a multi-step agent run when a step fails outside a supervised window?", "capability": "Design multi-step agent work with durable state, bounded retries, explicit handoffs, spend and time budgets, and a finite set of terminal dispositions.", "objectives": [ "Persist run state at step boundaries so a failure resumes rather than restarts.", "Bound retries, elapsed time, and spend so an unproductive run terminates instead of looping.", "Define every handoff as an explicit contract naming the receiving owner and the passed state.", "Enumerate terminal dispositions so no run ends in an undefined state.", "Bound queue depth and admission so backlogged work is refused rather than accumulated." ], "knowledgeEntryIds": [ "durable-execution", "orchestrator-supervisor", "agent-handoff", "goal-exit-condition", "multi-agent-system" ], "insightIds": [ "harness-engineering-is-platform-engineering", "own-the-speed" ], "patternIds": [ "durable-run-ledger" ], "visualIds": [ "evidence-decision-spine" ], "blueprintIds": [ "transactional-agent-with-approval" ], "toolPaths": [ "/control-plane-lab" ], "knowledgeSourceIds": [ "anthropic-long-running-harness", "openai-symphony", "anthropic-managed-agents", "a2a-v1" ], "practitionerIds": [], "lessons": [ { "id": "platform-orchestration-durable", "title": "What must survive a crash", "format": "read", "minutes": 16, "internalPath": "/agentic-engineering/durable-execution", "purpose": "Decide what state must survive a crash before designing multi-step runs." }, { "id": "platform-orchestration-shape-compare", "title": "Supervisor, handoff, and queue models", "format": "compare", "minutes": 18, "internalPath": "/agentic-engineering/orchestrator-supervisor", "purpose": "Choose a coordination shape from its failure behavior." }, { "id": "platform-orchestration-budget-practice", "title": "Set budgets and terminal dispositions", "format": "practice", "minutes": 32, "internalPath": "/control-plane-lab", "purpose": "Remove the undefined ending that turns a stuck run unbounded, in simulation." } ], "exercise": { "brief": "Design the run ledger for one multi-step agent workflow and walk it through a deliberate mid-run failure.", "constraints": [ "Persist state at every step boundary", "Set a retry ceiling, a time budget, and a spend budget", "Make every handoff name the receiving owner and the state it carries", "Enumerate terminal dispositions with no residual open state", "Walk one deliberate failure through the design and record the resume path", "State the queue depth at which new work is refused rather than accumulated" ], "deliverable": "A run ledger design with budgets, handoff contracts, terminal dispositions, and a recorded failure walkthrough.", "verifier": "A reviewer stops the run at an arbitrary step and states from the ledger alone what happened, what resumes, and what is now owed to a human.", "evidence": [ "Run ledger schema", "Budget settings", "Handoff contracts", "Terminal disposition list", "Failure walkthrough record" ] }, "exitCriteria": [ "Can explain why an unbounded retry is a cost and safety failure rather than resilience", "Can show that every run reaches one of a finite set of recorded endings" ] }, { "id": "platform-verifier-plane", "index": "06", "title": "Design the verifier plane", "drivingQuestion": "At which moments can a check still stop the work, and which checks only record what already happened?", "capability": "Design independently owned preventive, inline, gate, and continuous checks so that authority to stop the work exists before the point of consequence.", "objectives": [ "Place each check by control timing rather than by tooling convenience.", "Keep verifier ownership separate from the agent and the team whose work it challenges.", "Combine deterministic checks with model-based challenge instead of choosing one exclusively.", "Route disagreement to a named human rather than resolving it automatically." ], "knowledgeEntryIds": [ "multi-agent-verification", "observability-control", "guardrail", "deterministic-containment", "eval-driven-development", "human-accountability" ], "insightIds": [ "verification-is-a-plane-not-a-phase" ], "patternIds": [ "generator-verifier-separation" ], "visualIds": [ "governed-agentic-stack", "evidence-decision-spine" ], "blueprintIds": [ "consequential-member-impact-agent" ], "toolPaths": [ "/assurance-case", "/control-plane-lab" ], "knowledgeSourceIds": [ "nist-zero-trust", "anthropic-evals", "opentelemetry-genai", "nist-deployed-monitoring" ], "practitionerIds": [], "lessons": [ { "id": "platform-verifier-plane-read", "title": "Verification is a plane, not a phase", "format": "read", "minutes": 15, "internalPath": "/insights/verification-is-a-plane-not-a-phase", "purpose": "Separate the four control timings before choosing any checking tool." }, { "id": "platform-verifier-control-compare", "title": "A check that records against a check that refuses", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/observability-control", "purpose": "Locate which layers hold block authority and which only report after the fact." }, { "id": "platform-verifier-placement-practice", "title": "Place four checks around one run", "format": "practice", "minutes": 32, "internalPath": "/assurance-case", "purpose": "Test whether stopping authority exists before the moment of consequence." } ], "exercise": { "brief": "Design the verifier plane for one agent workload and show which checks can still refuse the work.", "constraints": [ "Place at least one preventive, one inline, and one gate check", "Name an owner for each check who is not the builder of the work", "State block authority and escalation behavior for each check", "Label the continuous layer as observing rather than blocking" ], "deliverable": "A verifier plane map with owners, triggers, scope, block authority, and escalation paths.", "verifier": "An independent reviewer submits a deliberately non-compliant candidate and records which check refuses it and which only reports it afterward.", "evidence": [ "Verifier plane map", "Owner assignments", "Block authority statement", "Non-compliant candidate walkthrough", "Escalation record" ] }, "exitCriteria": [ "Can explain why continuous monitoring is the last line and not the control", "Can show that no check in the plane is owned by the work it challenges" ] }, { "id": "platform-evidence-and-lifecycle", "index": "07", "title": "Close the evidence and lifecycle loop", "drivingQuestion": "When an agent's output is questioned a year later, what record answers the question without asking the people who were there?", "capability": "Design one addressable record that links runs, findings, approvals, releases, incidents, recertification, and retirement.", "objectives": [ "Give every run, finding, and decision a stable identifier that later records can reference.", "Attach provenance to a release so the exact candidate, verifier, and approver stay recoverable.", "Reopen standing when an incident, drift signal, or recertification date arrives.", "Define retirement so a decommissioned agent's record remains readable." ], "knowledgeEntryIds": [ "evidence-engineering", "continuous-recertification-retirement", "agent-incident-response", "agent-estate-governance", "observability", "human-accountability" ], "insightIds": [ "evidence-before-autonomy" ], "patternIds": [ "evidence-first-delivery", "durable-run-ledger" ], "visualIds": [ "evidence-decision-spine", "risk-adaptive-autonomy-field" ], "blueprintIds": [ "consequential-member-impact-agent" ], "toolPaths": [ "/agent-estate", "/assurance-case" ], "knowledgeSourceIds": [ "slsa-provenance", "nist-ai-rmf", "microsoft-agent-lifecycle", "opentelemetry-genai" ], "practitionerIds": [], "lessons": [ { "id": "platform-evidence-record", "title": "What a record must carry", "format": "read", "minutes": 16, "internalPath": "/agentic-engineering/evidence-engineering", "purpose": "Define the record before the run, not after the context is gone." }, { "id": "platform-evidence-logs-compare", "title": "Telemetry volume against decision evidence", "format": "compare", "minutes": 18, "internalPath": "/insights/evidence-before-autonomy", "purpose": "Separate the stream nobody reads from the records a decision rests on." }, { "id": "platform-evidence-trace-practice", "title": "Trace one release back to its intent", "format": "practice", "minutes": 30, "internalPath": "/agent-estate", "purpose": "Walk the spine backwards until a link breaks." } ], "exercise": { "brief": "Build the evidence spine for one released agent change and walk it backwards from the release to the original intent.", "constraints": [ "Give every run, finding, and approval a stable identifier", "Record the named human who authorized the release and the condition that reverses it", "Set a recertification date and the signals that reopen standing earlier", "State what remains readable after the agent is retired" ], "deliverable": "An evidence spine with identifiers, named approvals, recertification triggers, and retirement handling.", "verifier": "A reviewer picks any released change and reconstructs its intent, verifier findings, and named approver without contacting the people who built it.", "evidence": [ "Evidence spine map", "Identifier scheme", "Named approval record", "Recertification trigger list", "Retirement and retention rule" ] }, "exitCriteria": [ "Can explain why a log stream is not evidence", "Can name the signals that reopen an agent's standing before its scheduled review date" ] }, { "id": "lead-measure-the-whole-system", "index": "01", "title": "Measure the system, not the activity", "drivingQuestion": "Which numbers show that delivery improved, and which only show that people and agents generated more?", "capability": "Build a measurement set that separates local generation activity from delivered outcomes, stability, learning, and human cognitive load.", "objectives": [ "Distinguish activity counts from throughput, stability, and outcome signals.", "Pair every speed measure with a stability, quality, or human-load measure.", "State the failure each measure is meant to detect and the blind spot it carries.", "Reject measures that no named person will act on." ], "knowledgeEntryIds": [ "velocity-sickness", "organizational-absorption-capacity", "software-factory", "observability", "evidence-engineering" ], "insightIds": [ "own-the-speed" ], "patternIds": [ "durable-run-ledger" ], "visualIds": [ "evidence-decision-spine" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/adoption-workbench", "/enterprise-aisdlc" ], "knowledgeSourceIds": [ "space-developer-productivity", "google-dora-ai-development", "metr-developer-productivity-rct", "ai-engineer-velocity-sickness" ], "practitionerIds": [], "lessons": [ { "id": "lead-measure-velocity-sickness", "title": "Velocity sickness", "format": "read", "minutes": 14, "internalPath": "/agentic-engineering/velocity-sickness", "purpose": "Name the failure this path exists to prevent before choosing a single measure." }, { "id": "lead-measure-activity-versus-outcome", "title": "Compare activity measures with system measures", "format": "compare", "minutes": 20, "internalPath": "/insights/own-the-speed", "purpose": "Place each candidate measure against what it can detect and what it will miss." }, { "id": "lead-measure-assemble-set", "title": "Assemble a balanced measurement set", "format": "practice", "minutes": 35, "internalPath": "/adoption-workbench", "purpose": "Turn candidate measures into one defined, decision-linked set." } ], "exercise": { "brief": "Build a measurement set for one delivery group that reports delivered outcomes, stability, learning, and human load alongside generation volume.", "constraints": [ "Pair every speed measure with a stability or human-load measure", "Name the decision each measure is meant to inform", "Record the source system and collection method for each measure", "Exclude any measure no named person will act on", "Do not report individual output counts as a performance rating" ], "deliverable": "A measurement set with definitions, sources, paired counter-measures, and a blind-spot register.", "verifier": "A reviewer outside the delivery group checks whether any measure in the set could rise while delivered outcomes, stability, or human load got worse.", "evidence": [ "Measure definitions", "Collection sources and dates", "Paired counter-measures", "Blind-spot register", "Decision map" ] }, "exitCriteria": [ "Can explain why merged pull requests are not delivered value", "Can defend each measure in the set against a plausible gaming strategy" ] }, { "id": "lead-find-the-absorption-constraint", "index": "02", "title": "Find the absorption constraint", "drivingQuestion": "Where does generated work stop moving, and what does that queue cost?", "capability": "Map the decision, review, verification, integration, release, and operating queues that determine how much generated work becomes delivered value.", "objectives": [ "Trace one work item from request to operated outcome and record every wait state.", "Record wait time separately from active time at each handoff.", "Count rework loops as separate passes through a queue.", "Name the single constraint that limits verified throughput before proposing capacity anywhere else." ], "knowledgeEntryIds": [ "organizational-absorption-capacity", "software-factory", "human-accountability", "durable-execution", "observability" ], "insightIds": [ "own-the-speed" ], "patternIds": [ "durable-run-ledger" ], "visualIds": [ "evidence-decision-spine" ], "blueprintIds": [ "first-bounded-coding-agent" ], "toolPaths": [ "/adoption-workbench" ], "knowledgeSourceIds": [ "dora-wip-limits", "github-agent-pr-review", "google-dora-ai-development" ], "practitionerIds": [], "lessons": [ { "id": "lead-constraint-absorption-capacity", "title": "Organizational absorption capacity", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/organizational-absorption-capacity", "purpose": "Define the capacity model this module measures a real value stream against." }, { "id": "lead-constraint-factory-or-amplifier", "title": "Compare a production system with a queue amplifier", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/software-factory", "purpose": "Separate repeatable production from generation that only lengthens downstream queues." }, { "id": "lead-constraint-trace-work-items", "title": "Trace work items end to end", "format": "practice", "minutes": 40, "internalPath": "/adoption-workbench", "purpose": "Produce the timeline evidence a constraint claim must rest on." } ], "exercise": { "brief": "Trace three recently completed work items through every queue between request and operated outcome, then locate the constraint that limits verified throughput.", "constraints": [ "Use completed work items with real timestamps", "Record wait time separately from active time", "Name the owner of each queue", "Count each rework loop as a separate pass", "Do not propose added capacity until the constraint is named" ], "deliverable": "A value-stream map with wait time, rework counts, and one named constraint supported by the traced timelines.", "verifier": "The owner of each mapped queue checks the recorded wait times and rework counts against their own records and challenges any figure they cannot reproduce.", "evidence": [ "Work-item timelines", "Queue inventory", "Wait and active time split", "Rework log", "Constraint statement with its basis" ] }, "exitCriteria": [ "Can locate a constraint from recorded evidence rather than intuition", "Can explain why relieving a non-constraint queue does not raise delivered throughput" ] }, { "id": "lead-redesign-work-and-attention", "index": "03", "title": "Redesign work and attention", "drivingQuestion": "How should work be sized, sequenced, and packaged so human judgment lands where it changes the outcome?", "capability": "Set batch size, work limits, decision packets, and protected review time from measured capacity so human authority stays real under higher generation speed.", "objectives": [ "Size work so one unit can be reviewed and verified inside a single attention window.", "Set work-in-process limits from measured review and verification capacity.", "Package a decision so the accountable person can rule on it without reconstructing the work.", "Schedule protected decision and review time as a standing commitment." ], "knowledgeEntryIds": [ "human-accountability", "organizational-absorption-capacity", "intent-engineering", "goal-exit-condition", "evidence-engineering" ], "insightIds": [ "own-the-speed", "evidence-before-autonomy" ], "patternIds": [ "evidence-first-delivery", "bounded-delegation-envelope" ], "visualIds": [ "evidence-decision-spine" ], "blueprintIds": [ "transactional-agent-with-approval" ], "toolPaths": [ "/enterprise-aisdlc" ], "knowledgeSourceIds": [ "dora-small-batches", "dora-wip-limits", "github-agent-pr-review" ], "practitionerIds": [], "lessons": [ { "id": "lead-attention-human-accountability", "title": "Human accountability under delegation", "format": "read", "minutes": 14, "internalPath": "/agentic-engineering/human-accountability", "purpose": "Establish what a human decision must contain before work is redesigned around it." }, { "id": "lead-attention-decision-packet", "title": "Write one decision packet", "format": "practice", "minutes": 30, "internalPath": "/enterprise-aisdlc", "purpose": "Turn a live decision into a packet a named person can rule on without a walkthrough." }, { "id": "lead-attention-batch-versus-capacity", "title": "Compare batch sizes against review capacity", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/organizational-absorption-capacity", "purpose": "Derive a defensible work limit from measured capacity rather than preference." } ], "exercise": { "brief": "Redesign one group's work intake so batch size, work limits, and decision packets match the review and decision capacity measured in the previous module.", "constraints": [ "Derive the work-in-process limit from measured review capacity", "Size every batch so it is reviewable in one sitting", "State the question, options, evidence, and named decider in each decision packet", "Schedule protected decision time before raising intake", "Change nothing upstream of the constraint until the new limits run for one cycle" ], "deliverable": "A revised intake design with batch rules, a work-in-process limit and its basis, a decision packet template, and a protected review schedule.", "verifier": "A reviewer who did not write the packet rules on one sample packet using only its contents, and reports any question they could not answer from it.", "evidence": [ "Batch sizing rule", "Work-in-process limit and its basis", "Decision packet template", "Completed sample packet", "Protected time schedule" ] }, "exitCriteria": [ "Can set a work limit from measured capacity rather than preference", "Can explain how an unreviewable batch removes human authority in practice while leaving it on the org chart" ] }, { "id": "lead-scale-evidence-and-verification", "index": "04", "title": "Scale verification and evidence with generation", "drivingQuestion": "What has to grow alongside generation capacity so more output does not quietly mean less assurance?", "capability": "Plan independent verification and release evidence that grows with generation capacity, with named ownership and a stop condition when capacity is exceeded.", "objectives": [ "Separate builder self-checks from independently owned challenge.", "Size verifier ownership and evidence capture against expected release volume.", "Define the evidence each release must carry before it can advance.", "Set a stop condition tied to a measured verification backlog." ], "knowledgeEntryIds": [ "multi-agent-verification", "evidence-engineering", "eval-driven-development", "observability-control", "human-accountability" ], "insightIds": [ "verification-is-a-plane-not-a-phase", "evidence-before-autonomy" ], "patternIds": [ "generator-verifier-separation", "evidence-first-delivery" ], "visualIds": [ "evidence-decision-spine", "governed-agentic-stack" ], "blueprintIds": [ "transactional-agent-with-approval" ], "toolPaths": [ "/assurance-case" ], "knowledgeSourceIds": [ "nist-zero-trust", "anthropic-evals", "slsa-provenance" ], "practitionerIds": [], "lessons": [ { "id": "lead-assurance-verification-plane", "title": "Verification is a plane, not a phase", "format": "read", "minutes": 15, "internalPath": "/insights/verification-is-a-plane-not-a-phase", "purpose": "Establish where challenge belongs before sizing how much of it a delivery group needs." }, { "id": "lead-assurance-self-versus-independent", "title": "Compare self-review with independent challenge", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/multi-agent-verification", "purpose": "Locate the independence boundary a capacity plan has to respect." }, { "id": "lead-assurance-size-the-plane", "title": "Size the verifier plane against release volume", "format": "practice", "minutes": 35, "internalPath": "/assurance-case", "purpose": "Convert the independence boundary into named ownership and a countable capacity assumption." } ], "exercise": { "brief": "Plan the verification and evidence capacity required for a stated increase in release volume, then state the condition under which the group must stop raising volume.", "constraints": [ "Name an owner for every check, and never the builder of the work it challenges", "State the trigger, scope, and block authority of each check", "Define the evidence each release must carry before it advances", "Tie the stop condition to a measured backlog rather than a judgment call", "Describe every planned check as a design proposal, not an operating control" ], "deliverable": "A verification capacity plan with named verifier ownership, per-release evidence requirements, capacity assumptions, and a stop condition.", "verifier": "An assurance owner outside the delivery group checks that each proposed check has a trigger, a scope, authority to block, and a recorded place its disposition would land, and rejects any check whose owner also produces the work.", "evidence": [ "Verifier inventory", "Ownership map", "Per-release evidence requirements", "Capacity assumptions", "Stop condition and its measure" ] }, "exitCriteria": [ "Can explain why builder-run checks are not independent assurance", "Can state the release volume at which current verification capacity fails and what happens then" ] }, { "id": "lead-bind-governance-to-execution", "index": "05", "title": "Bind governance to execution", "drivingQuestion": "What makes a written policy actually reach the moment an agent is about to act?", "capability": "Connect delegated standing, policy decisions, intervention, and named disposition so written rules reach the point where an action can be refused, held, or escalated.", "objectives": [ "Trace one policy statement from document text to a specific decision point.", "Distinguish guidance that advises from a control that can refuse an action.", "Bind each agent to a sponsor, owner, scope, and expiry.", "Record who intervened, on what basis, and with what disposition." ], "knowledgeEntryIds": [ "agent-identity", "runtime-policy-enforcement", "risk-tiered-autonomy", "agent-estate-governance", "human-accountability" ], "insightIds": [ "every-agent-is-a-governed-principal", "evidence-before-autonomy" ], "patternIds": [ "risk-adaptive-autonomy", "agent-identity-lifecycle" ], "visualIds": [ "risk-adaptive-autonomy-field", "governed-agentic-stack" ], "blueprintIds": [ "transactional-agent-with-approval", "consequential-member-impact-agent" ], "toolPaths": [ "/policy-pack", "/control-plane-lab" ], "knowledgeSourceIds": [ "nist-ai-rmf", "microsoft-agent-risk-governance", "opa-policy-enforcement" ], "practitionerIds": [], "lessons": [ { "id": "lead-governance-governed-principal", "title": "Every agent is a governed principal", "format": "read", "minutes": 14, "internalPath": "/insights/every-agent-is-a-governed-principal", "purpose": "Establish the standing question every policy trace depends on." }, { "id": "lead-governance-policy-versus-control", "title": "Compare written policy with an enforceable control", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/runtime-policy-enforcement", "purpose": "Learn to tell a statement of intent from a decision point that can refuse an action." }, { "id": "lead-governance-rehearse-intervention", "title": "Rehearse a hold, a correction, and a named disposition", "format": "practice", "minutes": 35, "internalPath": "/control-plane-lab", "purpose": "Practise finite intervention steps in a study setting while keeping the simulation boundary explicit." } ], "exercise": { "brief": "Take three policy statements your organization already publishes and trace each one to the point where an action could be refused, held, or escalated.", "constraints": [ "Use existing published policy text rather than statements written for the exercise", "Mark each statement as guidance or enforceable and give the reason", "Name the sponsor, owner, scope, and expiry of every agent identity in the trace", "Record the disposition of each rehearsed intervention", "Describe every traced control as a design proposal, not an operating control" ], "deliverable": "A policy-to-enforcement trace with agent standing records and rehearsed intervention dispositions.", "verifier": "A risk or compliance reviewer outside the delivery group checks each trace for a specific decision point and rejects any statement that only restates intent.", "evidence": [ "Policy statements as published", "Enforcement point per statement", "Agent standing records", "Intervention rehearsal log", "Disposition records" ] }, "exitCriteria": [ "Can tell guidance apart from a control that can refuse an action", "Can name the standing, scope, and expiry of every agent in a delegated workflow" ] }, { "id": "lead-learn-from-production", "index": "06", "title": "Close the loop from production", "drivingQuestion": "How does what production shows change what the organization asks for next?", "capability": "Turn production outcomes, incidents, corrections, and drift into revised intent, durable evaluation cases, and reopened autonomy decisions.", "objectives": [ "Compare the outcome an initiative promised with what production shows.", "Convert an incident or correction into a named evaluation case that would have failed before the fix.", "Detect drift in behaviour, context sources, and dependencies over time.", "Reopen scope and autonomy decisions from operating evidence rather than on a calendar." ], "knowledgeEntryIds": [ "agent-incident-response", "continuous-recertification-retirement", "observability-control", "eval-driven-development", "evidence-engineering", "intent-engineering" ], "insightIds": [ "evidence-before-autonomy", "verification-is-a-plane-not-a-phase" ], "patternIds": [ "durable-run-ledger", "risk-adaptive-autonomy" ], "visualIds": [ "evidence-decision-spine", "risk-adaptive-autonomy-field" ], "blueprintIds": [ "consequential-member-impact-agent" ], "toolPaths": [ "/agent-estate", "/assurance-case" ], "knowledgeSourceIds": [ "nist-deployed-monitoring", "openai-monitoring", "microsoft-agent-lifecycle" ], "practitionerIds": [], "lessons": [ { "id": "lead-production-signals-that-decide", "title": "Signals that describe and signals that decide", "format": "read", "minutes": 15, "internalPath": "/agentic-engineering/observability-control", "purpose": "Separate monitoring that reports the system from signals that should reopen a decision." }, { "id": "lead-production-correction-versus-scope", "title": "Compare a one-time correction with a standing scope change", "format": "compare", "minutes": 20, "internalPath": "/agentic-engineering/continuous-recertification-retirement", "purpose": "Decide when a finding is a fix and when it should narrow an agent's scope or retire it." }, { "id": "lead-production-finding-to-eval", "title": "Convert one production finding into an evaluation case", "format": "practice", "minutes": 35, "internalPath": "/assurance-case", "purpose": "Feed operating evidence back into the standing verification checks." } ], "exercise": { "brief": "Take one recorded production incident or missed outcome and carry it back into revised intent, a new evaluation case, and an autonomy decision.", "constraints": [ "Use a real recorded event rather than a hypothetical", "Write the evaluation case so it fails against the original behaviour", "State what scope or autonomy change the finding justifies", "Name the human who owns the resulting decision", "Keep unresolved findings visible rather than closing them to finish the exercise" ], "deliverable": "A learning record linking one production finding to revised intent, a new evaluation case, and an autonomy decision with a named owner.", "verifier": "A reviewer independent of the correction runs the new evaluation case against the original behaviour and confirms it fails before the change and passes after.", "evidence": [ "Production finding", "Root-cause note", "New evaluation case", "Revised intent statement", "Autonomy decision record", "Open-finding register" ] }, "exitCriteria": [ "Can convert an operating failure into a durable check that would have caught it", "Can name which production signals should reopen an autonomy decision and who rules on it" ] } ], "capstones": [ { "id": "bounded-feature-evidence-dossier", "title": "Bounded feature evidence dossier", "brief": "Carry one reversible feature from clarified intent through an end-to-end slice, bounded agent implementation, independent challenge, evidence packaging, and a named human release disposition.", "deliverables": [ "Intent and decision record", "System and module maps", "Tracer-slice plan", "Test and evaluation receipts", "Independent verifier disposition", "Human release decision and rollback condition" ], "verification": [ "Every artifact names the exact candidate it describes", "The verifier is independent of the builder", "Known limitations and unresolved findings remain visible", "The release decision is human, named, and reversible" ], "completionBoundary": "Progress and completion are self-directed learning records. They are not certification, professional standing, production evidence, or authorization to deploy an agentic system." }, { "id": "private-inference-decision-dossier", "title": "Private inference decision dossier", "brief": "Carry one real workload of your own from a traced data path through an ownership decision, a measured hardware and serving plan, an evidence-backed adaptation position, and a governed model register, ending in a named human disposition with a stated reversal condition.", "deliverables": [ "Data-path map with a store-level access and retention register", "Ownership decision record naming the binding constraint and the rejected alternatives", "Measured hardware fit report and serving operating plan for the chosen level, plus the measurements that ruled out the alternative", "Failure rehearsal record with the observed recovery time and remaining gaps", "Frozen evaluation set and the adaptation position it supports", "Model register with owner, exact version, license, provenance, and expiry", "Named human disposition stating the condition that would reverse it" ], "verification": [ "Every measured figure names the hardware, the exact model version, and the date it was taken.", "A reviewer independent of the author reproduces one measurement and one configuration claim from the dossier.", "Unverified license, provenance, and retention entries stay marked as unverified rather than resolved by assumption.", "The privacy boundary is stated as what the configuration shows, not as what the deployment location implies.", "The final disposition is recorded by a named human decision owner rather than by the learner's agent, and states the condition that would reverse it." ], "completionBoundary": "Progress and completion are self-directed learning records. They are not certification, professional standing, production evidence, or authorization to deploy an agentic system." }, { "id": "agent-platform-service-dossier", "title": "Agent platform service dossier", "brief": "Carry one real agent workload across every platform service in this path, from substrate and identity through context, tools, orchestration, verification, and evidence, and close it with a named human disposition rather than an agent decision.", "deliverables": [ "Substrate contract with isolation boundaries, credential handling, and recovery procedure", "Identity record naming purpose, sponsor, accountable owner, scoped permissions, and expiry", "Context manifest with source versions, trust classes, and retention rules", "Typed tool contract set separating inspect, propose, execute, and commit capability", "Run ledger with budgets, handoff contracts, and a finite terminal disposition list", "Verifier plane map with owners, triggers, block authority, and escalation paths", "Evidence spine linking the run to findings, approval, recertification trigger, and retirement handling" ], "verification": [ "A reviewer who did not build the dossier traces one run from intent to disposition using only the written artifacts", "Every service in the slice names an owner who is neither the agent nor the person who designed it", "An out-of-scope action is refused by a written contract rather than by model cooperation", "Unresolved gaps, residual exposure, and untested assumptions remain visible instead of being resolved on paper", "The final disposition on the slice is recorded by a named human, not by the learner's agent" ], "completionBoundary": "Progress and completion are self-directed learning records. They are not certification, professional standing, production evidence, or authorization to deploy an agentic system." }, { "id": "delivery-operating-review", "title": "AI-native delivery operating review", "brief": "Produce one operating review for a real delivery group. Measure the whole system, locate the absorption constraint from recorded wait time, redesign batch size and decision packets, size verification and evidence against expected release volume, trace three policy statements to enforcement points, and carry one production finding back into intent. Close the review with a named human disposition and the conditions that would reverse it.", "deliverables": [ "Balanced measurement set with definitions, collection sources, and known blind spots", "Value-stream map with wait time, rework counts, and one named constraint", "Revised intake design with batch rules, work-in-process limits, and a decision packet template", "Verification and evidence capacity plan with named verifier ownership and a stop condition", "Policy-to-enforcement trace with agent standing records and rehearsed intervention dispositions", "Production learning record linking one finding to revised intent and a new evaluation case", "Named human disposition stating the decision, its basis, and what would reverse it" ], "verification": [ "Every figure in the review traces to a named source system and a collection date", "The named constraint is supported by recorded wait time and rework counts rather than assertion", "Verification ownership is independent of the people and agents generating the work", "Every proposed control is labelled as a design proposal, not as an operating control", "The final disposition is recorded by a named human decision owner rather than the learner's agent, and states the conditions that would reverse it" ], "completionBoundary": "Progress and completion are self-directed learning records. They are not certification, professional standing, production evidence, or authorization to deploy an agentic system." } ], "practitioners": [ { "id": "matt-pocock", "name": "Matt Pocock", "verifiedDescription": "Developer educator and creator of AI Hero and Total TypeScript.", "canonicalUrl": "https://www.aihero.dev/", "relationship": "independent-source", "sourceIds": [ "matt-pocock-software-fundamentals", "matt-pocock-skills-workflow", "matt-pocock-wayfinder", "matt-pocock-software-design-playlist", "matt-pocock-skills-repository", "matt-pocock-ai-hero" ], "attributedPositions": [ "Software fundamentals and strategic engineering judgment become more important as code generation accelerates.", "Teams should reach shared understanding before turning an idea into a specification and implementation plan.", "Vertical slices, tight test feedback, deliberate module boundaries, and fresh phase contexts make agent-assisted delivery more inspectable.", "Large, uncertain efforts benefit from resolving decision questions before they become implementation tickets." ], "aisdlcExtensions": [ "A fresh-context review is a correlated quality check; independent verification requires separately owned challenge with an appropriate model lineage or deterministic method.", "Specifications and skills guide work but do not confer authority; governance assigns standing and the control plane makes constraints binding.", "Consequential release requires attributable evidence and a named human decision authority beyond an implementation workflow." ], "attributionBoundary": "Matt Pocock is included as an independent practitioner source. AISDLC owns this synthesis and does not represent him as a contributor, partner, endorser, or approver.", "reviewedAt": "2026-08-10" } ], "practitionerSources": [ { "id": "matt-pocock-software-fundamentals", "practitionerId": "matt-pocock", "relationshipToPractitioner": "speaker", "title": "“Software Fundamentals Matter More Than Ever” — Matt Pocock", "publisher": "AI Engineer", "url": "https://www.youtube.com/watch?v=v4F1gFy-hqg", "kind": "video", "claimRole": "Practitioner perspective", "summary": "A practitioner argument for retaining strategic engineering judgment, inspectable code, test feedback, deliberate module design, and human comprehension as generation accelerates.", "limitations": "This is an attributed conference position, not a controlled study, consensus standard, universal productivity law, or enterprise release method.", "rights": "link-or-standard-embed-only", "youtubeId": "v4F1gFy-hqg" }, { "id": "matt-pocock-skills-workflow", "practitionerId": "matt-pocock", "relationshipToPractitioner": "creator", "title": "mattpocock/skills: A complete AI Coding workflow, end-to-end", "publisher": "Matt Pocock", "url": "https://www.youtube.com/watch?v=M6mYodf0dJM", "kind": "video", "claimRole": "Practitioner perspective", "summary": "A walkthrough connecting idea interrogation, durable context, specifications, tracer-slice tickets, implementation, test-driven feedback, and code review.", "limitations": "This demonstrates one practitioner workflow and tool ecosystem; it does not establish independent verification, governed authority, or fitness for every team and codebase.", "rights": "link-or-standard-embed-only", "youtubeId": "M6mYodf0dJM" }, { "id": "matt-pocock-wayfinder", "practitionerId": "matt-pocock", "relationshipToPractitioner": "creator", "title": "/wayfinder: Nothing is too big to plan anymore", "publisher": "Matt Pocock", "url": "https://www.youtube.com/watch?v=F3lL98Pj90o", "kind": "video", "claimRole": "Practitioner perspective", "summary": "A practitioner model for turning large, foggy efforts into explicit decision tickets whose resolved map can later become a specification and implementation plan.", "limitations": "Wayfinder is an evolving practitioner method. Its planning map does not replace accountable product decisions, architecture review, implementation controls, or release evidence.", "rights": "link-or-standard-embed-only", "youtubeId": "F3lL98Pj90o" }, { "id": "matt-pocock-software-design-playlist", "practitionerId": "matt-pocock", "relationshipToPractitioner": "featured-in-third-party-collection", "title": "Software Design playlist", "publisher": "Software Design · Harry Phung", "url": "https://www.youtube.com/playlist?list=PLLjGHW6KybtUXPYeFEBTM6pEVKemaGXXT", "kind": "playlist", "claimRole": "Practitioner perspective", "summary": "A third-party public collection by Harry Phung / Software Design that includes the supplied Matt Pocock talk and other software-design material for editorial discovery.", "limitations": "Matt Pocock is featured in the collection but is not represented as its curator or publisher. A playlist is a discovery surface, not evidence for a claim; individual items require review and attribution.", "rights": "link-or-standard-embed-only" }, { "id": "matt-pocock-skills-repository", "practitionerId": "matt-pocock", "relationshipToPractitioner": "author-maintainer", "title": "Skills for Real Engineers", "publisher": "GitHub · mattpocock/skills", "url": "https://github.com/mattpocock/skills", "kind": "repository", "claimRole": "Open-source implementation reference", "summary": "A public, evolving repository of agent skills and workflow documentation for engineering, planning, testing, review, teaching, and maintenance.", "limitations": "The repository is a practitioner implementation, not an enterprise control standard. Substantial reuse must preserve its MIT copyright and permission notice.", "rights": "mit-source-with-attribution" }, { "id": "matt-pocock-ai-hero", "practitionerId": "matt-pocock", "relationshipToPractitioner": "canonical-home", "title": "AI Hero", "publisher": "AI Hero", "url": "https://www.aihero.dev/", "kind": "website", "claimRole": "Practitioner home", "summary": "Matt Pocock’s canonical public teaching site for AI engineering articles, videos, workshops, skills, and cohort information.", "limitations": "Public descriptions can establish attribution and available offerings, but paid lessons, course exercises, visual assets, and brand identity are not AISDLC content.", "rights": "link-or-standard-embed-only" } ], "resolvedReferences": { "knowledge": [ { "id": "ai-agent", "title": "AI agent", "url": "https://aisdlc.ai/agentic-engineering/ai-agent" }, { "id": "agentic-engineering", "title": "Agentic engineering", "url": "https://aisdlc.ai/agentic-engineering/agentic-engineering" }, { "id": "agent-loop", "title": "Agent loop", "url": "https://aisdlc.ai/agentic-engineering/agent-loop" }, { "id": "harness-engineering", "title": "Harness engineering", "url": "https://aisdlc.ai/agentic-engineering/harness-engineering" }, { "id": "execution-substrate", "title": "Agent execution substrate", "url": "https://aisdlc.ai/agentic-engineering/execution-substrate" }, { "id": "intent-engineering", "title": "Intent engineering", "url": "https://aisdlc.ai/agentic-engineering/intent-engineering" }, { "id": "context-engineering", "title": "Context engineering", "url": "https://aisdlc.ai/agentic-engineering/context-engineering" }, { "id": "working-state", "title": "Working state", "url": "https://aisdlc.ai/agentic-engineering/working-state" }, { "id": "goal-exit-condition", "title": "Goal and exit condition", "url": "https://aisdlc.ai/agentic-engineering/goal-exit-condition" }, { "id": "durable-memory", "title": "Durable project memory", "url": "https://aisdlc.ai/agentic-engineering/durable-memory" }, { "id": "instruction-data-trust-boundary", "title": "Instruction–data trust boundary", "url": "https://aisdlc.ai/agentic-engineering/instruction-data-trust-boundary" }, { "id": "durable-execution", "title": "Durable execution", "url": "https://aisdlc.ai/agentic-engineering/durable-execution" }, { "id": "organizational-absorption-capacity", "title": "Organizational absorption capacity", "url": "https://aisdlc.ai/agentic-engineering/organizational-absorption-capacity" }, { "id": "eval-driven-development", "title": "Eval-driven development", "url": "https://aisdlc.ai/agentic-engineering/eval-driven-development" }, { "id": "structured-output", "title": "Structured output", "url": "https://aisdlc.ai/agentic-engineering/structured-output" }, { "id": "llm-as-judge", "title": "LLM as judge", "url": "https://aisdlc.ai/agentic-engineering/llm-as-judge" }, { "id": "observability", "title": "Observability", "url": "https://aisdlc.ai/agentic-engineering/observability" }, { "id": "multi-agent-verification", "title": "Independent verifier systems", "url": "https://aisdlc.ai/agentic-engineering/multi-agent-verification" }, { "id": "human-accountability", "title": "Human accountability", "url": "https://aisdlc.ai/agentic-engineering/human-accountability" }, { "id": "evidence-engineering", "title": "Evidence engineering", "url": "https://aisdlc.ai/agentic-engineering/evidence-engineering" }, { "id": "deterministic-containment", "title": "Deterministic containment", "url": "https://aisdlc.ai/agentic-engineering/deterministic-containment" }, { "id": "software-factory", "title": "Agentic software factory", "url": "https://aisdlc.ai/agentic-engineering/software-factory" }, { "id": "runtime-policy-enforcement", "title": "Runtime policy enforcement", "url": "https://aisdlc.ai/agentic-engineering/runtime-policy-enforcement" }, { "id": "agent-incident-response", "title": "Agent incident response", "url": "https://aisdlc.ai/agentic-engineering/agent-incident-response" }, { "id": "continuous-recertification-retirement", "title": "Continuous recertification & retirement", "url": "https://aisdlc.ai/agentic-engineering/continuous-recertification-retirement" }, { "id": "private-ai", "title": "Private AI", "url": "https://aisdlc.ai/agentic-engineering/private-ai" }, { "id": "agent-identity", "title": "Agent identity & delegated authority", "url": "https://aisdlc.ai/agentic-engineering/agent-identity" }, { "id": "open-weight-model", "title": "Open-weight model", "url": "https://aisdlc.ai/agentic-engineering/open-weight-model" }, { "id": "self-hosted-inference", "title": "Self-hosted inference", "url": "https://aisdlc.ai/agentic-engineering/self-hosted-inference" }, { "id": "model-adaptation", "title": "Model adaptation", "url": "https://aisdlc.ai/agentic-engineering/model-adaptation" }, { "id": "large-language-model", "title": "Large language model", "url": "https://aisdlc.ai/agentic-engineering/large-language-model" }, { "id": "model-routing", "title": "Model routing", "url": "https://aisdlc.ai/agentic-engineering/model-routing" }, { "id": "local-inference", "title": "Local inference", "url": "https://aisdlc.ai/agentic-engineering/local-inference" }, { "id": "quantization", "title": "Quantization", "url": "https://aisdlc.ai/agentic-engineering/quantization" }, { "id": "context-window", "title": "Context window", "url": "https://aisdlc.ai/agentic-engineering/context-window" }, { "id": "inference-serving", "title": "Inference serving", "url": "https://aisdlc.ai/agentic-engineering/inference-serving" }, { "id": "observability-control", "title": "Observability & control", "url": "https://aisdlc.ai/agentic-engineering/observability-control" }, { "id": "sandbox", "title": "Sandbox", "url": "https://aisdlc.ai/agentic-engineering/sandbox" }, { "id": "parameter-efficient-fine-tuning", "title": "Parameter-efficient fine-tuning", "url": "https://aisdlc.ai/agentic-engineering/parameter-efficient-fine-tuning" }, { "id": "retrieval-augmented-generation", "title": "Retrieval-augmented generation", "url": "https://aisdlc.ai/agentic-engineering/retrieval-augmented-generation" }, { "id": "agent-estate-governance", "title": "Agent estate governance", "url": "https://aisdlc.ai/agentic-engineering/agent-estate-governance" }, { "id": "risk-tiered-autonomy", "title": "Risk-tiered autonomy", "url": "https://aisdlc.ai/agentic-engineering/risk-tiered-autonomy" }, { "id": "context-hygiene", "title": "Context hygiene", "url": "https://aisdlc.ai/agentic-engineering/context-hygiene" }, { "id": "tools-and-protocols", "title": "Tools, skills & protocols", "url": "https://aisdlc.ai/agentic-engineering/tools-and-protocols" }, { "id": "tool-calling", "title": "Tool calling", "url": "https://aisdlc.ai/agentic-engineering/tool-calling" }, { "id": "model-context-protocol", "title": "Model Context Protocol", "url": "https://aisdlc.ai/agentic-engineering/model-context-protocol" }, { "id": "orchestrator-supervisor", "title": "Orchestrator or supervisor agent", "url": "https://aisdlc.ai/agentic-engineering/orchestrator-supervisor" }, { "id": "agent-handoff", "title": "Agent handoff", "url": "https://aisdlc.ai/agentic-engineering/agent-handoff" }, { "id": "multi-agent-system", "title": "Multi-agent system", "url": "https://aisdlc.ai/agentic-engineering/multi-agent-system" }, { "id": "guardrail", "title": "Guardrail", "url": "https://aisdlc.ai/agentic-engineering/guardrail" }, { "id": "velocity-sickness", "title": "Velocity sickness", "url": "https://aisdlc.ai/agentic-engineering/velocity-sickness" } ], "insights": [ { "id": "the-model-is-not-the-architecture", "title": "The model is not the architecture", "url": "https://aisdlc.ai/insights/the-model-is-not-the-architecture" }, { "id": "context-is-a-runtime-system", "title": "Context is a runtime system", "url": "https://aisdlc.ai/insights/context-is-a-runtime-system" }, { "id": "harness-engineering-is-platform-engineering", "title": "Harness engineering is platform engineering", "url": "https://aisdlc.ai/insights/harness-engineering-is-platform-engineering" }, { "id": "own-the-speed", "title": "Own the speed", "url": "https://aisdlc.ai/insights/own-the-speed" }, { "id": "verification-is-a-plane-not-a-phase", "title": "Verification is a plane, not a phase", "url": "https://aisdlc.ai/insights/verification-is-a-plane-not-a-phase" }, { "id": "evidence-before-autonomy", "title": "Evidence before autonomy", "url": "https://aisdlc.ai/insights/evidence-before-autonomy" }, { "id": "every-agent-is-a-governed-principal", "title": "Every agent is a governed principal", "url": "https://aisdlc.ai/insights/every-agent-is-a-governed-principal" } ], "patterns": [ { "id": "bounded-delegation-envelope", "title": "Bounded delegation envelope", "url": "https://aisdlc.ai/agentic-engineering/patterns#bounded-delegation-envelope" }, { "id": "context-manifest", "title": "Context manifest", "url": "https://aisdlc.ai/agentic-engineering/patterns#context-manifest" }, { "id": "durable-run-ledger", "title": "Durable run ledger", "url": "https://aisdlc.ai/agentic-engineering/patterns#durable-run-ledger" }, { "id": "generator-verifier-separation", "title": "Generator–verifier separation", "url": "https://aisdlc.ai/agentic-engineering/patterns#generator-verifier-separation" }, { "id": "evidence-first-delivery", "title": "Evidence-first delivery", "url": "https://aisdlc.ai/agentic-engineering/patterns#evidence-first-delivery" }, { "id": "risk-adaptive-autonomy", "title": "Risk-adaptive autonomy", "url": "https://aisdlc.ai/agentic-engineering/patterns#risk-adaptive-autonomy" }, { "id": "agent-identity-lifecycle", "title": "Agent identity lifecycle", "url": "https://aisdlc.ai/agentic-engineering/patterns#agent-identity-lifecycle" }, { "id": "tool-contract-firewall", "title": "Tool-contract firewall", "url": "https://aisdlc.ai/agentic-engineering/patterns#tool-contract-firewall" } ], "visuals": [ { "id": "governed-agentic-stack", "title": "The governed agentic stack", "url": "https://aisdlc.ai/agentic-engineering/visuals#governed-agentic-stack" }, { "id": "evidence-decision-spine", "title": "The evidence and decision spine", "url": "https://aisdlc.ai/agentic-engineering/visuals#evidence-decision-spine" }, { "id": "risk-adaptive-autonomy-field", "title": "The autonomy field", "url": "https://aisdlc.ai/agentic-engineering/visuals#risk-adaptive-autonomy-field" } ], "blueprints": [ { "id": "first-bounded-coding-agent", "title": "First bounded coding agent", "url": "https://aisdlc.ai/agentic-engineering/blueprints#first-bounded-coding-agent" }, { "id": "transactional-agent-with-approval", "title": "Transactional agent with approval authority", "url": "https://aisdlc.ai/agentic-engineering/blueprints#transactional-agent-with-approval" }, { "id": "regulated-read-only-agent", "title": "Regulated read-only knowledge agent", "url": "https://aisdlc.ai/agentic-engineering/blueprints#regulated-read-only-agent" }, { "id": "consequential-member-impact-agent", "title": "Consequential member-impact agent", "url": "https://aisdlc.ai/agentic-engineering/blueprints#consequential-member-impact-agent" } ], "sources": [ { "id": "anthropic-building-effective-agents", "title": "Building effective agents", "publisher": "Anthropic", "url": "https://www.anthropic.com/engineering/building-effective-agents" }, { "id": "openai-practical-agent-guide", "title": "A practical guide to building agents", "publisher": "OpenAI", "url": "https://openai.com/business/guides-and-resources/a-practical-guide-to-building-ai-agents/" }, { "id": "openai-harness-engineering", "title": "Harness engineering: leveraging Codex in an agent-first world", "publisher": "OpenAI", "url": "https://openai.com/index/harness-engineering/" }, { "id": "github-spec-driven", "title": "Spec-driven development with AI", "publisher": "GitHub", "url": "https://github.blog/ai-and-ml/generative-ai/spec-driven-development-with-ai-get-started-with-a-new-open-source-toolkit/" }, { "id": "github-spec-persistence", "title": "Spec persistence models", "publisher": "GitHub Spec Kit", "url": "https://github.github.com/spec-kit/concepts/spec-persistence.html" }, { "id": "anthropic-context", "title": "Effective context engineering for AI agents", "publisher": "Anthropic", "url": "https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents" }, { "id": "anthropic-long-running-harness", "title": "Effective harnesses for long-running agents", "publisher": "Anthropic", "url": "https://www.anthropic.com/engineering/effective-harnesses-for-long-running-agents" }, { "id": "dora-small-batches", "title": "Working in small batches", "publisher": "DORA", "url": "https://dora.dev/capabilities/working-in-small-batches/" }, { "id": "dora-wip-limits", "title": "Work in process limits", "publisher": "DORA", "url": "https://dora.dev/capabilities/wip-limits/" }, { "id": "anthropic-evals", "title": "Demystifying evals for AI agents", "publisher": "Anthropic", "url": "https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents" }, { "id": "nist-benchmark-evals", "title": "Towards best practices for automated benchmark evaluations", "publisher": "NIST", "url": "https://www.nist.gov/news-events/news/2026/01/towards-best-practices-automated-benchmark-evaluations" }, { "id": "llm-judge-position-bias", "title": "Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge", "publisher": "arXiv", "url": "https://arxiv.org/abs/2406.07791" }, { "id": "nist-zero-trust", "title": "Zero Trust Architecture (NIST SP 800-207)", "publisher": "NIST", "url": "https://csrc.nist.gov/pubs/sp/800/207/final" }, { "id": "google-dora-ai-development", "title": "DORA 2025 State of AI-assisted Software Development Report", "publisher": "Google Research / DORA", "url": "https://research.google/pubs/dora-2025-state-of-ai-assisted-software-development-report/" }, { "id": "github-agent-pr-review", "title": "Agent pull requests are everywhere. Here's how to review them", "publisher": "GitHub", "url": "https://github.blog/ai-and-ml/generative-ai/agent-pull-requests-are-everywhere-heres-how-to-review-them/" }, { "id": "openai-monitoring", "title": "How we monitor internal coding agents for misalignment", "publisher": "OpenAI", "url": "https://openai.com/index/how-we-monitor-internal-coding-agents-misalignment/" }, { "id": "owasp-memory", "title": "Memory Is a Feature. It Is Also an Attack Surface", "publisher": "OWASP GenAI Security Project", "url": "https://genai.owasp.org/2026/05/13/memory-is-a-feature-it-is-also-an-attack-surface/" }, { "id": "opentelemetry-genai", "title": "GenAI observability", "publisher": "OpenTelemetry", "url": "https://opentelemetry.io/blog/2026/genai-observability/" }, { "id": "openai-gpt-oss", "title": "Introducing gpt-oss", "publisher": "OpenAI", "url": "https://openai.com/index/introducing-gpt-oss/" }, { "id": "openai-gpt-oss-model-card", "title": "gpt-oss-120b & gpt-oss-20b Model Card", "publisher": "OpenAI", "url": "https://openai.com/index/gpt-oss-model-card/" }, { "id": "nist-ai-rmf", "title": "Artificial Intelligence Risk Management Framework 1.0", "publisher": "NIST", "url": "https://www.nist.gov/itl/ai-risk-management-framework" }, { "id": "llama-cpp-local-inference", "title": "llama.cpp", "publisher": "ggml.org", "url": "https://github.com/ggml-org/llama.cpp" }, { "id": "ollama-local-only", "title": "Ollama FAQ", "publisher": "Ollama", "url": "https://docs.ollama.com/faq" }, { "id": "hf-quantization", "title": "Quantization overview", "publisher": "Hugging Face", "url": "https://huggingface.co/docs/transformers/quantization/overview" }, { "id": "artificial-analysis-methodology", "title": "Artificial Analysis Intelligence Benchmarking Methodology", "publisher": "Artificial Analysis", "url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking" }, { "id": "vllm-serving", "title": "OpenAI-Compatible Server", "publisher": "vLLM", "url": "https://docs.vllm.ai/en/latest/serving/online_serving/openai_compatible_server/" }, { "id": "google-agent-substrate", "title": "Agent Sandbox on GKE and Agent Substrate", "publisher": "Google Cloud", "url": "https://cloud.google.com/blog/products/containers-kubernetes/bringing-you-agent-sandbox-on-gke-and-agent-substrate" }, { "id": "nist-deployed-monitoring", "title": "Challenges in monitoring deployed AI systems", "publisher": "NIST CAISI", "url": "https://www.nist.gov/publications/challenges-monitoring-deployed-ai-systems-center-ai-standards-and-innovation" }, { "id": "hf-peft-methods", "title": "Parameter efficient fine-tuning methods", "publisher": "Hugging Face", "url": "https://huggingface.co/docs/peft/main/methods/overview" }, { "id": "lora-paper", "title": "LoRA: Low-Rank Adaptation of Large Language Models", "publisher": "arXiv / ICLR", "url": "https://arxiv.org/abs/2106.09685" }, { "id": "hf-fine-tuning", "title": "Fine-tuning", "publisher": "Hugging Face", "url": "https://huggingface.co/docs/transformers/en/training" }, { "id": "slsa-provenance", "title": "SLSA provenance specification v1.2", "publisher": "SLSA", "url": "https://slsa.dev/spec/v1.2/provenance" }, { "id": "microsoft-agent-lifecycle", "title": "Manage the agent lifecycle", "publisher": "Microsoft Agentic Center of Excellence", "url": "https://learn.microsoft.com/en-us/agents/center-of-excellence/agent-lifecycle" }, { "id": "anthropic-managed-agents", "title": "Scaling Managed Agents: Decoupling the brain from the hands", "publisher": "Anthropic", "url": "https://www.anthropic.com/engineering/managed-agents" }, { "id": "owasp-agentic-top10", "title": "OWASP Top 10 for Agentic Applications 2026", "publisher": "OWASP GenAI Security Project", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/" }, { "id": "nist-agent-identity", "title": "Identity and Authority for Software Agents concept paper", "publisher": "NIST NCCoE", "url": "https://www.nccoe.nist.gov/sites/default/files/2026-02/accelerating-the-adoption-of-software-and-ai-agent-identity-and-authorization-concept-paper.pdf" }, { "id": "microsoft-agent-identities", "title": "What are agent identities?", "publisher": "Microsoft Entra", "url": "https://learn.microsoft.com/en-us/entra/agent-id/what-are-agent-identities" }, { "id": "microsoft-agent-owners-sponsors", "title": "Administrative relationships in Microsoft Entra Agent ID", "publisher": "Microsoft Entra Agent ID", "url": "https://learn.microsoft.com/en-us/entra/agent-id/agent-owners-sponsors-managers" }, { "id": "microsoft-agent-id-governance", "title": "Governing Agent Identities", "publisher": "Microsoft Entra ID Governance", "url": "https://learn.microsoft.com/en-us/entra/id-governance/agent-id-governance-overview" }, { "id": "mcp-enterprise-auth", "title": "Enterprise-Managed Authorization for MCP", "publisher": "Model Context Protocol", "url": "https://blog.modelcontextprotocol.io/posts/enterprise-managed-auth/" }, { "id": "mcp-architecture", "title": "Model Context Protocol architecture overview", "publisher": "Model Context Protocol", "url": "https://modelcontextprotocol.io/docs/2026-07-28/learn/architecture" }, { "id": "opa-policy-enforcement", "title": "OPA Management APIs and Architecture", "publisher": "Open Policy Agent", "url": "https://www.openpolicyagent.org/docs/management-introduction" }, { "id": "openai-symphony", "title": "An open-source spec for Codex orchestration: Symphony", "publisher": "OpenAI", "url": "https://openai.com/index/open-source-codex-orchestration-symphony/" }, { "id": "a2a-v1", "title": "Agent2Agent Protocol v1.0", "publisher": "A2A Project", "url": "https://a2a-protocol.org/latest/announcing-1.0/" }, { "id": "space-developer-productivity", "title": "The SPACE of Developer Productivity: There's more to it than you think", "publisher": "Microsoft Research / ACM Queue", "url": "https://www.microsoft.com/en-us/research/publication/the-space-of-developer-productivity-theres-more-to-it-than-you-think/" }, { "id": "metr-developer-productivity-rct", "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity", "publisher": "METR", "url": "https://metr.org/blog/2025-07-10-early-2025-ai-experienced-os-dev-study/" }, { "id": "ai-engineer-velocity-sickness", "title": "Velocity Sickness: What Happens When Your Whole Team Gets 10x Faster", "publisher": "Matt Dailey · AI Engineer World's Fair", "url": "https://www.youtube.com/watch?v=Kz4QJmNrVXU" }, { "id": "microsoft-agent-risk-governance", "title": "Govern agents by risk", "publisher": "Microsoft Agentic Center of Excellence", "url": "https://learn.microsoft.com/en-us/agents/center-of-excellence/govern-agents-risk" } ] } }