[{"data":1,"prerenderedAt":965},["ShallowReactive",2],{"navigation_docs":3,"-build-plugins-evals":320,"-build-plugins-evals-surround":960},[4,35,69,109,133,193,228,262,305],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20,25,30],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-ph-house",{"title":16,"path":17,"stem":18,"icon":19},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F2.quickstart","i-ph-rocket-launch",{"title":21,"path":22,"stem":23,"icon":24},"Concepts","\u002Fgetting-started\u002Fconcepts","1.getting-started\u002F3.concepts","i-ph-book-open",{"title":26,"path":27,"stem":28,"icon":29},"Install","\u002Fgetting-started\u002Finstall","1.getting-started\u002F4.install","i-ph-download-simple",{"title":31,"path":32,"stem":33,"icon":34},"What works today","\u002Fgetting-started\u002Fstatus","1.getting-started\u002F5.status","i-ph-list-checks",{"title":21,"icon":6,"path":36,"stem":37,"children":38,"page":6},"\u002Fconcepts","2.concepts",[39,44,49,54,59,64],{"title":40,"path":41,"stem":42,"icon":43},"Architecture","\u002Fconcepts\u002Farchitecture","2.concepts\u002F1.architecture","i-ph-planet",{"title":45,"path":46,"stem":47,"icon":48},"Jobs and durability","\u002Fconcepts\u002Fjobs","2.concepts\u002F2.jobs","i-ph-infinity",{"title":50,"path":51,"stem":52,"icon":53},"Routing","\u002Fconcepts\u002Frouting","2.concepts\u002F3.routing","i-ph-path",{"title":55,"path":56,"stem":57,"icon":58},"Security model","\u002Fconcepts\u002Fsecurity","2.concepts\u002F4.security","i-ph-shield-check",{"title":60,"path":61,"stem":62,"icon":63},"Tenants","\u002Fconcepts\u002Ftenants","2.concepts\u002F5.tenants","i-ph-buildings",{"title":65,"path":66,"stem":67,"icon":68},"Temporal","\u002Fconcepts\u002Ftemporal","2.concepts\u002F6.temporal","i-ph-clock",{"title":70,"icon":6,"path":71,"stem":72,"children":73,"page":6},"Run a fleet","\u002Ffleet","3.fleet",[74,79,84,89,94,99,104],{"title":75,"path":76,"stem":77,"icon":78},"Dev mode","\u002Ffleet\u002Fdev-mode","3.fleet\u002F1.dev-mode","i-ph-laptop",{"title":80,"path":81,"stem":82,"icon":83},"Run a cluster","\u002Ffleet\u002Fcluster","3.fleet\u002F2.cluster","i-ph-hard-drives",{"title":85,"path":86,"stem":87,"icon":88},"Join nodes","\u002Ffleet\u002Fjoin-nodes","3.fleet\u002F3.join-nodes","i-ph-plugs-connected",{"title":90,"path":91,"stem":92,"icon":93},"Manage nodes","\u002Ffleet\u002Fmanage-nodes","3.fleet\u002F4.manage-nodes","i-ph-list-bullets",{"title":95,"path":96,"stem":97,"icon":98},"Secrets","\u002Ffleet\u002Fsecrets","3.fleet\u002F5.secrets","i-ph-key",{"title":100,"path":101,"stem":102,"icon":103},"High availability","\u002Ffleet\u002Fhigh-availability","3.fleet\u002F6.high-availability","i-ph-copy",{"title":105,"path":106,"stem":107,"icon":108},"Upgrades, backups and restores","\u002Ffleet\u002Fupgrades","3.fleet\u002F7.upgrades","i-ph-arrows-clockwise",{"title":110,"icon":6,"path":111,"stem":112,"children":113,"page":6},"Use plugins","\u002Fplugins","4.plugins",[114,119,124,128],{"title":115,"path":116,"stem":117,"icon":118},"What a plugin is","\u002Fplugins\u002Foverview","4.plugins\u002F1.overview","i-ph-puzzle-piece",{"title":120,"path":121,"stem":122,"icon":123},"Install and update","\u002Fplugins\u002Finstall","4.plugins\u002F2.install","i-ph-package",{"title":125,"path":126,"stem":127,"icon":123},"Existing MCP packages","\u002Fplugins\u002Fpackages","4.plugins\u002F3.packages",{"title":129,"path":130,"stem":131,"icon":132},"The catalog","\u002Fplugins\u002Fcatalog","4.plugins\u002F4.catalog","i-ph-books",{"title":134,"icon":6,"path":135,"stem":136,"children":137,"page":6},"Build plugins","\u002Fbuild-plugins","5.build-plugins",[138,143,148,153,158,163,168,173,178,183,188],{"title":139,"path":140,"stem":141,"icon":142},"Your first plugin","\u002Fbuild-plugins\u002Ffirst-plugin","5.build-plugins\u002F1.first-plugin","i-ph-hammer",{"title":144,"path":145,"stem":146,"icon":147},"Harnesses","\u002Fbuild-plugins\u002Fharnesses","5.build-plugins\u002F10.harnesses","i-ph-cpu",{"title":149,"path":150,"stem":151,"icon":152},"Example: support triage","\u002Fbuild-plugins\u002Fexample-support-triage","5.build-plugins\u002F11.example-support-triage","i-ph-lifebuoy",{"title":154,"path":155,"stem":156,"icon":157},"Tools","\u002Fbuild-plugins\u002Ftools","5.build-plugins\u002F2.tools","i-ph-wrench",{"title":159,"path":160,"stem":161,"icon":162},"Agents","\u002Fbuild-plugins\u002Fagents","5.build-plugins\u002F3.agents","i-ph-robot",{"title":164,"path":165,"stem":166,"icon":167},"Skills","\u002Fbuild-plugins\u002Fskills","5.build-plugins\u002F4.skills","i-ph-graduation-cap",{"title":169,"path":170,"stem":171,"icon":172},"Develop and test","\u002Fbuild-plugins\u002Fdevelop-and-test","5.build-plugins\u002F5.develop-and-test","i-ph-flask",{"title":174,"path":175,"stem":176,"icon":177},"Publish and version","\u002Fbuild-plugins\u002Fpublish","5.build-plugins\u002F6.publish","i-ph-tag",{"title":179,"path":180,"stem":181,"icon":182},"Hooks","\u002Fbuild-plugins\u002Fhooks","5.build-plugins\u002F7.hooks","i-ph-anchor",{"title":184,"path":185,"stem":186,"icon":187},"Flows","\u002Fbuild-plugins\u002Fflows","5.build-plugins\u002F8.flows","i-ph-flow-arrow",{"title":189,"path":190,"stem":191,"icon":192},"Evals and quality gates","\u002Fbuild-plugins\u002Fevals","5.build-plugins\u002F9.evals","i-ph-seal-check",{"title":194,"icon":6,"path":195,"stem":196,"children":197,"page":6},"Run agents","\u002Fagents","6.agents",[198,203,208,213,218,223],{"title":199,"path":200,"stem":201,"icon":202},"Run jobs","\u002Fagents\u002Frun-jobs","6.agents\u002F1.run-jobs","i-ph-play",{"title":204,"path":205,"stem":206,"icon":207},"Models and keys","\u002Fagents\u002Fmodels-and-keys","6.agents\u002F2.models-and-keys","i-ph-sparkle",{"title":209,"path":210,"stem":211,"icon":212},"Delegation","\u002Fagents\u002Fdelegation","6.agents\u002F3.delegation","i-ph-git-fork",{"title":214,"path":215,"stem":216,"icon":217},"Contracts and limits","\u002Fagents\u002Fcontracts-and-limits","6.agents\u002F4.contracts-and-limits","i-ph-ruler",{"title":219,"path":220,"stem":221,"icon":222},"Runtime API","\u002Fagents\u002Fruntime-api","6.agents\u002F5.runtime-api","i-ph-plug",{"title":224,"path":225,"stem":226,"icon":227},"Approvals","\u002Fagents\u002Fapprovals","6.agents\u002F6.approvals","i-ph-hand-palm",{"title":229,"icon":6,"path":230,"stem":231,"children":232,"page":6},"Monitor","\u002Fmonitor","7.monitor",[233,238,243,248,253,258],{"title":234,"path":235,"stem":236,"icon":237},"Web UI","\u002Fmonitor\u002Fweb-ui","7.monitor\u002F0.web-ui","i-ph-monitor",{"title":239,"path":240,"stem":241,"icon":242},"Cost and usage","\u002Fmonitor\u002Fcost-and-usage","7.monitor\u002F1.cost-and-usage","i-ph-coins",{"title":244,"path":245,"stem":246,"icon":247},"Node health","\u002Fmonitor\u002Fnodes","7.monitor\u002F2.nodes","i-ph-heartbeat",{"title":249,"path":250,"stem":251,"icon":252},"Job history","\u002Fmonitor\u002Fhistory","7.monitor\u002F3.history","i-ph-clock-counter-clockwise",{"title":254,"path":255,"stem":256,"icon":257},"Tracing and metrics","\u002Fmonitor\u002Fobservability","7.monitor\u002F4.observability","i-ph-chart-line",{"title":259,"path":260,"stem":261,"icon":152},"Troubleshooting","\u002Fmonitor\u002Ftroubleshooting","7.monitor\u002F5.troubleshooting",{"title":263,"icon":6,"path":264,"stem":265,"children":266,"page":6},"Reference","\u002Freference","8.reference",[267,281,286,290,295,300],{"title":268,"icon":269,"path":270,"stem":271,"children":272,"page":6},"CLI","i-ph-terminal-window","\u002Freference\u002Fcli","8.reference\u002F1.cli",[273,277],{"title":274,"path":275,"stem":276},"orchestrator-zero","\u002Freference\u002Fcli\u002Forchestrator-zero","8.reference\u002F1.cli\u002F1.orchestrator-zero",{"title":278,"path":279,"stem":280},"orchestrator-zero-node","\u002Freference\u002Fcli\u002Forchestrator-zero-node","8.reference\u002F1.cli\u002F2.orchestrator-zero-node",{"title":282,"path":283,"stem":284,"icon":285},"oz0-plugin.yaml","\u002Freference\u002Fplugin-manifest","8.reference\u002F2.plugin-manifest","i-ph-file-code",{"title":287,"path":288,"stem":289,"icon":162},"Agent definition","\u002Freference\u002Fagent-definition","8.reference\u002F3.agent-definition",{"title":291,"path":292,"stem":293,"icon":294},"Ports","\u002Freference\u002Fports","8.reference\u002F4.ports","i-ph-network",{"title":296,"path":297,"stem":298,"icon":299},"Environment variables","\u002Freference\u002Fenvironment","8.reference\u002F5.environment","i-ph-brackets-curly",{"title":301,"path":302,"stem":303,"icon":304},"Files and directories","\u002Freference\u002Ffiles","8.reference\u002F6.files","i-ph-tree-structure",{"title":306,"icon":6,"path":307,"stem":308,"children":309,"page":6},"About","\u002Fabout","9.about",[310,315],{"title":311,"path":312,"stem":313,"icon":314},"Licensing","\u002Fabout\u002Flicensing","9.about\u002F1.licensing","i-ph-scales",{"title":316,"path":317,"stem":318,"icon":319},"About these docs","\u002Fabout\u002Fcontributing-docs","9.about\u002F2.contributing-docs","i-ph-pencil-simple",{"id":321,"title":189,"body":322,"description":953,"extension":954,"links":955,"meta":956,"navigation":957,"path":190,"seo":958,"stem":191,"__hash__":959},"docs\u002F5.build-plugins\u002F9.evals.md",{"type":323,"value":324,"toc":947},"minimark",[325,334,522,533,538,541,661,667,671,712,720,731,735,738,776,779,783,787,790,827,850,856,862,868,875,878,935,943],[326,327,328,329,333],"p",{},"Evals are test cases in your plugin: an input for an agent or a flow, and what a good answer looks like. ",[330,331,332],"code",{},"orchestrator-zero plugin test"," runs them on a dev server and a node of its own, and model answers can be recorded once and replayed after, so tests in CI are fast, free and repeatable.",[335,336,342],"pre",{"className":337,"code":338,"filename":339,"language":340,"meta":341,"style":341},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","agent: greeter            # or flow: \u003Cname>\nthreshold: 0.8            # the share of cases that must pass; all of them by default\ncases:\n  - name: greets by name\n    input: Please greet Ada\n    expect:\n      contains: [\"Hello, Ada\"]\n      cel: \"calls \u003C= 3 && cost_usd \u003C 0.02\"\n  - name: refuses an empty name\n    input: { name: \"\" }\n    expect:\n      status: failed\n      error_contains: InvalidInput\n","evals\u002Fgreetings.yaml","yaml","",[330,343,344,365,380,389,403,414,422,444,461,473,493,500,511],{"__ignoreMap":341},[345,346,349,353,357,361],"span",{"class":347,"line":348},"line",1,[345,350,352],{"class":351},"swJcz","agent",[345,354,356],{"class":355},"sMK4o",":",[345,358,360],{"class":359},"sfazB"," greeter",[345,362,364],{"class":363},"sHwdD","            # or flow: \u003Cname>\n",[345,366,368,371,373,377],{"class":347,"line":367},2,[345,369,370],{"class":351},"threshold",[345,372,356],{"class":355},[345,374,376],{"class":375},"sbssI"," 0.8",[345,378,379],{"class":363},"            # the share of cases that must pass; all of them by default\n",[345,381,383,386],{"class":347,"line":382},3,[345,384,385],{"class":351},"cases",[345,387,388],{"class":355},":\n",[345,390,392,395,398,400],{"class":347,"line":391},4,[345,393,394],{"class":355},"  -",[345,396,397],{"class":351}," name",[345,399,356],{"class":355},[345,401,402],{"class":359}," greets by name\n",[345,404,406,409,411],{"class":347,"line":405},5,[345,407,408],{"class":351},"    input",[345,410,356],{"class":355},[345,412,413],{"class":359}," Please greet Ada\n",[345,415,417,420],{"class":347,"line":416},6,[345,418,419],{"class":351},"    expect",[345,421,388],{"class":355},[345,423,425,428,430,433,436,439,441],{"class":347,"line":424},7,[345,426,427],{"class":351},"      contains",[345,429,356],{"class":355},[345,431,432],{"class":355}," [",[345,434,435],{"class":355},"\"",[345,437,438],{"class":359},"Hello, Ada",[345,440,435],{"class":355},[345,442,443],{"class":355},"]\n",[345,445,447,450,452,455,458],{"class":347,"line":446},8,[345,448,449],{"class":351},"      cel",[345,451,356],{"class":355},[345,453,454],{"class":355}," \"",[345,456,457],{"class":359},"calls \u003C= 3 && cost_usd \u003C 0.02",[345,459,460],{"class":355},"\"\n",[345,462,464,466,468,470],{"class":347,"line":463},9,[345,465,394],{"class":355},[345,467,397],{"class":351},[345,469,356],{"class":355},[345,471,472],{"class":359}," refuses an empty name\n",[345,474,476,478,480,483,485,487,490],{"class":347,"line":475},10,[345,477,408],{"class":351},[345,479,356],{"class":355},[345,481,482],{"class":355}," {",[345,484,397],{"class":351},[345,486,356],{"class":355},[345,488,489],{"class":355}," \"\"",[345,491,492],{"class":355}," }\n",[345,494,496,498],{"class":347,"line":495},11,[345,497,419],{"class":351},[345,499,388],{"class":355},[345,501,503,506,508],{"class":347,"line":502},12,[345,504,505],{"class":351},"      status",[345,507,356],{"class":355},[345,509,510],{"class":359}," failed\n",[345,512,514,517,519],{"class":347,"line":513},13,[345,515,516],{"class":351},"      error_contains",[345,518,356],{"class":355},[345,520,521],{"class":359}," InvalidInput\n",[326,523,524,525,528,529,532],{},"List eval files or folders in the manifest with ",[330,526,527],{},"evals: [evals\u002F]",". ",[330,530,531],{},"plugin lint"," checks them, CEL included.",[534,535,537],"h2",{"id":536},"what-a-case-can-expect","What a case can expect",[326,539,540],{},"Every expectation that is set must hold:",[542,543,544,557],"table",{},[545,546,547],"thead",{},[548,549,550,554],"tr",{},[551,552,553],"th",{},"Expectation",[551,555,556],{},"Holds when",[558,559,560,578,592,606,616,648],"tbody",{},[548,561,562,568],{},[563,564,565],"td",{},[330,566,567],{},"status",[563,569,570,571,574,575],{},"The job ends ",[330,572,573],{},"completed"," (the default) or ",[330,576,577],{},"failed",[548,579,580,585],{},[563,581,582],{},[330,583,584],{},"error_contains",[563,586,587,588,591],{},"The failed job's error contains this text (with ",[330,589,590],{},"status: failed",")",[548,593,594,603],{},[563,595,596,599,600],{},[330,597,598],{},"contains",", ",[330,601,602],{},"not_contains",[563,604,605],{},"The answer, as text (JSON for structured answers), contains each, or none, of these",[548,607,608,613],{},[563,609,610],{},[330,611,612],{},"equals",[563,614,615],{},"The answer is exactly this value",[548,617,618,623],{},[563,619,620],{},[330,621,622],{},"cel",[563,624,625,626,633,634,637,638,599,641,644,645],{},"This ",[627,628,632],"a",{"href":629,"rel":630},"https:\u002F\u002Fcel.dev",[631],"nofollow","CEL"," expression is true; it sees ",[330,635,636],{},"output"," (the answer), ",[330,639,640],{},"cost_usd",[330,642,643],{},"calls"," and ",[330,646,647],{},"seconds",[548,649,650,658],{},[563,651,652,599,655],{},[330,653,654],{},"max_cost_usd",[330,656,657],{},"max_seconds",[563,659,660],{},"The job cost and took no more than this",[326,662,663,664,666],{},"An eval passes when the share of passing cases reaches its ",[330,665,370],{},".",[534,668,670],{"id":669},"run-them","Run them",[335,672,677],{"className":673,"code":674,"filename":675,"language":676,"meta":341,"style":341},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","orchestrator-zero plugin test .\u002Fgreeter            # every eval of the plugin\norchestrator-zero plugin test .\u002Fgreeter --eval greetings\n","Terminal","bash",[330,678,679,696],{"__ignoreMap":341},[345,680,681,684,687,690,693],{"class":347,"line":348},[345,682,274],{"class":683},"sBMFI",[345,685,686],{"class":359}," plugin",[345,688,689],{"class":359}," test",[345,691,692],{"class":359}," .\u002Fgreeter",[345,694,695],{"class":363},"            # every eval of the plugin\n",[345,697,698,700,702,704,706,709],{"class":347,"line":367},[345,699,274],{"class":683},[345,701,686],{"class":359},[345,703,689],{"class":359},[345,705,692],{"class":359},[345,707,708],{"class":359}," --eval",[345,710,711],{"class":359}," greetings\n",[335,713,718],{"className":714,"code":716,"language":717,"meta":341},[715],"language-text","  PASS  greetings \u002F greets by name (replayed, 2 calls, $0.0011, 0.5 s)\n  FAIL  greetings \u002F says goodbye (replayed, 2 calls, $0.0011, 0.3 s): the answer does not contain \"Goodbye\"\ngreetings (greeter): 2 of 3 passed (67%), threshold 60%: ok; $0.0032, 1.2 s\n","text",[330,719,716],{"__ignoreMap":341},[326,721,722,723,726,727,730],{},"The command starts a dev server and a node, installs the plugin from its working tree, and runs the cases one at a time as jobs. It exits non-zero when an eval is below its threshold, so it can gate a pull request. ",[330,724,725],{},"--json"," prints the report for machines. It needs what ",[330,728,729],{},"orchestrator-zero dev"," needs: the node binary next to the CLI and a runtime bundle.",[534,732,734],{"id":733},"record-once-replay-after","Record once, replay after",[326,736,737],{},"Every model call of a case goes through the test server's gateway, and from there through a recorder in the CLI:",[739,740,741,762,768],"ul",{},[742,743,744,750,751,754,755,758,759,666],"li",{},[745,746,747],"strong",{},[330,748,749],{},"--record"," calls the real API for every case, with ",[330,752,753],{},"ANTHROPIC_API_KEY"," or ",[330,756,757],{},"OPENAI_API_KEY"," from your environment, and keeps each answer in ",[330,760,761],{},"evals\u002Fcassettes\u002F\u003Ceval>\u002F\u003Ccase>.jsonl",[742,763,764,767],{},[745,765,766],{},"Without flags",", a case that has a recording is answered from it, in order, without calling anyone and without a key; a case without one calls the real API.",[742,769,770,775],{},[745,771,772],{},[330,773,774],{},"--live"," ignores recordings.",[326,777,778],{},"Commit the recordings with the plugin: they hold what the model answered, never your key or the request's headers. When a prompt changes, a replay still answers in order and warns that the call differs from the recording; record again to bring it up to date. A case that makes more calls than its recording holds fails and says so. Costs in a replay are what the recorded calls cost.",[780,781,782],"note",{},"Grading by a rubric that another model scores, and running cases in parallel, come later.",[534,784,786],{"id":785},"quality-gates","Quality gates",[326,788,789],{},"When you install a version of a plugin that has evals, the edge runs them before the fleet gets the version, against your tenant's real models and keys:",[791,792,794,799,802,806,813,817,820,824],"steps",{"level":793},"4",[795,796,798],"h4",{"id":797},"the-version-waits-as-the-plugins-candidate","The version waits as the plugin's candidate",[326,800,801],{},"The fleet keeps the version it has. A plugin's first version waits too: its agents run once it passes.",[795,803,805],{"id":804},"one-node-tests-it","One node tests it",[326,807,808,809,812],{},"The edge picks an online node of the tenant that matches the plugin's selector and the labels its agents need, and prefers nodes labelled ",[330,810,811],{},"oz0.gate=true",". That node runs the new version instead of the installed one, on queues of its own: no fleet job reaches it, and no eval case reaches the fleet. While no such node is online, the gate waits for one.",[795,814,816],{"id":815},"the-evals-run-there","The evals run there",[326,818,819],{},"Every case runs as a job through the gateway, as in production: real models, real tools, real cost. Recordings are not used.",[795,821,823],{"id":822},"the-verdict","The verdict",[326,825,826],{},"The version passes when every eval reaches its threshold and passes at least the share of cases that the installed version passed at its own gate. It then becomes the installed version and rolls out, and its results become the bar for the next version. Otherwise it is stopped: the fleet keeps the installed version and the test node goes back to it.",[335,828,830],{"className":673,"code":829,"filename":675,"language":676,"meta":341,"style":341},"orchestrator-zero plugin install https:\u002F\u002Fgithub.com\u002Facme\u002Fteam --ref v0.2.0\n",[330,831,832],{"__ignoreMap":341},[345,833,834,836,838,841,844,847],{"class":347,"line":348},[345,835,274],{"class":683},[345,837,686],{"class":359},[345,839,840],{"class":359}," install",[345,842,843],{"class":359}," https:\u002F\u002Fgithub.com\u002Facme\u002Fteam",[345,845,846],{"class":359}," --ref",[345,848,849],{"class":359}," v0.2.0\n",[335,851,854],{"className":852,"code":853,"language":717,"meta":341},[715],"Testing team 0.2.0 (commit ff4d7eae2c6d…) in tenant default before the fleet gets it: its evals run on a test node.\nUntil it passes, the fleet keeps team 0.1.0.\nFollow with: orchestrator-zero plugin list (--no-gate installs without the evals)\n",[330,855,853],{"__ignoreMap":341},[326,857,858,861],{},[330,859,860],{},"plugin list"," and the Plugins page of the web UI show the gate: waiting, testing on a node, or how it ended and why, with each eval's result. A version that does worse reads like this:",[335,863,866],{"className":864,"code":865,"language":717,"meta":341},[715],"Gate of team 0.2.0: stopped: eval greetings: 0 of 1 cases passed (0%), below its threshold of 100%\n",[330,867,865],{"__ignoreMap":341},[326,869,870,871,874],{},"The audit log records every verdict. ",[330,872,873],{},"plugin install --no-gate"," installs a version at once, without running its evals.",[326,876,877],{},"Good to know:",[739,879,880,893,899,905,919,929],{},[742,881,882,885,886,888,889,892],{},[745,883,884],{},"A dedicated test node"," keeps gates away from the nodes that serve jobs: give it the label ",[330,887,811],{}," with its join token (",[330,890,891],{},"token create --label oz0.gate=true",") or when you accept it. While a gate runs, its node serves no fleet jobs of that plugin, so a tenant with a single node waits for the gate before those agents run again.",[742,894,895,898],{},[745,896,897],{},"A version that cannot start"," on the test node, or does not start within 10 minutes, is stopped. A case may run for 10 minutes.",[742,900,901,904],{},[745,902,903],{},"Children stay with the candidate:"," a flow's agent steps and an agent's delegations to agents of the same plugin run the new version on the test node; agents of other plugins answer from the fleet, in their installed versions.",[742,906,907,910,911,914,915,918],{},[745,908,909],{},"Installing again during a gate:"," the version under test again (a ",[330,912,913],{},"plugin update"," while it runs) lets its gate go on; a newer version replaces it and its gate starts over; the installed version again, or ",[330,916,917],{},"--no-gate",", cancels it. A version that was stopped is tested again when you install it again.",[742,920,921,928],{},[745,922,923,644,925],{},[330,924,729],{},[330,926,927],{},"plugin test"," install your working tree without a gate.",[742,930,931,934],{},[745,932,933],{},"Servers share the work:"," any server can drive a gate. If it stops, another one runs the gate again from the start within 30 seconds.",[780,936,937,938,942],{},"Rolling a plugin version out to canary nodes, next to the old version on real jobs and with the same verdict, is planned. ",[627,939,941],{"href":940},"\u002Ffleet\u002Fupgrades#roll-a-runtime-out-in-steps","Rollouts"," do this for the runtime today.",[944,945,946],"style",{},"html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}",{"title":341,"searchDepth":367,"depth":367,"links":948},[949,950,951,952],{"id":536,"depth":367,"text":537},{"id":669,"depth":367,"text":670},{"id":733,"depth":367,"text":734},{"id":785,"depth":367,"text":786},"Test cases for your agents and flows, run with recorded model answers, that stop a worse version.","md",null,{},{"icon":192},{"title":189,"description":953},"junTE6OUcXc2TcA8m1_XCPAi7hQeUgxaKHD_-wd3B58",[961,963],{"title":184,"path":185,"stem":186,"description":962,"icon":187,"children":-1},"Declarative steps that chain tools, agents, conditions and approvals.",{"title":199,"path":200,"stem":201,"description":964,"icon":202,"children":-1},"Start an agent, follow it, cancel it and list what ran.",1791135412014]