[{"data":1,"prerenderedAt":1205},["ShallowReactive",2],{"navigation_docs":3,"-fleet-upgrades":320,"-fleet-upgrades-surround":1200},[4,35,69,109,133,193,228,262,305],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20,25,30],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-ph-house",{"title":16,"path":17,"stem":18,"icon":19},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F2.quickstart","i-ph-rocket-launch",{"title":21,"path":22,"stem":23,"icon":24},"Concepts","\u002Fgetting-started\u002Fconcepts","1.getting-started\u002F3.concepts","i-ph-book-open",{"title":26,"path":27,"stem":28,"icon":29},"Install","\u002Fgetting-started\u002Finstall","1.getting-started\u002F4.install","i-ph-download-simple",{"title":31,"path":32,"stem":33,"icon":34},"What works today","\u002Fgetting-started\u002Fstatus","1.getting-started\u002F5.status","i-ph-list-checks",{"title":21,"icon":6,"path":36,"stem":37,"children":38,"page":6},"\u002Fconcepts","2.concepts",[39,44,49,54,59,64],{"title":40,"path":41,"stem":42,"icon":43},"Architecture","\u002Fconcepts\u002Farchitecture","2.concepts\u002F1.architecture","i-ph-planet",{"title":45,"path":46,"stem":47,"icon":48},"Jobs and durability","\u002Fconcepts\u002Fjobs","2.concepts\u002F2.jobs","i-ph-infinity",{"title":50,"path":51,"stem":52,"icon":53},"Routing","\u002Fconcepts\u002Frouting","2.concepts\u002F3.routing","i-ph-path",{"title":55,"path":56,"stem":57,"icon":58},"Security model","\u002Fconcepts\u002Fsecurity","2.concepts\u002F4.security","i-ph-shield-check",{"title":60,"path":61,"stem":62,"icon":63},"Tenants","\u002Fconcepts\u002Ftenants","2.concepts\u002F5.tenants","i-ph-buildings",{"title":65,"path":66,"stem":67,"icon":68},"Temporal","\u002Fconcepts\u002Ftemporal","2.concepts\u002F6.temporal","i-ph-clock",{"title":70,"icon":6,"path":71,"stem":72,"children":73,"page":6},"Run a fleet","\u002Ffleet","3.fleet",[74,79,84,89,94,99,104],{"title":75,"path":76,"stem":77,"icon":78},"Dev mode","\u002Ffleet\u002Fdev-mode","3.fleet\u002F1.dev-mode","i-ph-laptop",{"title":80,"path":81,"stem":82,"icon":83},"Run a cluster","\u002Ffleet\u002Fcluster","3.fleet\u002F2.cluster","i-ph-hard-drives",{"title":85,"path":86,"stem":87,"icon":88},"Join nodes","\u002Ffleet\u002Fjoin-nodes","3.fleet\u002F3.join-nodes","i-ph-plugs-connected",{"title":90,"path":91,"stem":92,"icon":93},"Manage nodes","\u002Ffleet\u002Fmanage-nodes","3.fleet\u002F4.manage-nodes","i-ph-list-bullets",{"title":95,"path":96,"stem":97,"icon":98},"Secrets","\u002Ffleet\u002Fsecrets","3.fleet\u002F5.secrets","i-ph-key",{"title":100,"path":101,"stem":102,"icon":103},"High availability","\u002Ffleet\u002Fhigh-availability","3.fleet\u002F6.high-availability","i-ph-copy",{"title":105,"path":106,"stem":107,"icon":108},"Upgrades, backups and restores","\u002Ffleet\u002Fupgrades","3.fleet\u002F7.upgrades","i-ph-arrows-clockwise",{"title":110,"icon":6,"path":111,"stem":112,"children":113,"page":6},"Use plugins","\u002Fplugins","4.plugins",[114,119,124,128],{"title":115,"path":116,"stem":117,"icon":118},"What a plugin is","\u002Fplugins\u002Foverview","4.plugins\u002F1.overview","i-ph-puzzle-piece",{"title":120,"path":121,"stem":122,"icon":123},"Install and update","\u002Fplugins\u002Finstall","4.plugins\u002F2.install","i-ph-package",{"title":125,"path":126,"stem":127,"icon":123},"Existing MCP packages","\u002Fplugins\u002Fpackages","4.plugins\u002F3.packages",{"title":129,"path":130,"stem":131,"icon":132},"The catalog","\u002Fplugins\u002Fcatalog","4.plugins\u002F4.catalog","i-ph-books",{"title":134,"icon":6,"path":135,"stem":136,"children":137,"page":6},"Build plugins","\u002Fbuild-plugins","5.build-plugins",[138,143,148,153,158,163,168,173,178,183,188],{"title":139,"path":140,"stem":141,"icon":142},"Your first plugin","\u002Fbuild-plugins\u002Ffirst-plugin","5.build-plugins\u002F1.first-plugin","i-ph-hammer",{"title":144,"path":145,"stem":146,"icon":147},"Harnesses","\u002Fbuild-plugins\u002Fharnesses","5.build-plugins\u002F10.harnesses","i-ph-cpu",{"title":149,"path":150,"stem":151,"icon":152},"Example: support triage","\u002Fbuild-plugins\u002Fexample-support-triage","5.build-plugins\u002F11.example-support-triage","i-ph-lifebuoy",{"title":154,"path":155,"stem":156,"icon":157},"Tools","\u002Fbuild-plugins\u002Ftools","5.build-plugins\u002F2.tools","i-ph-wrench",{"title":159,"path":160,"stem":161,"icon":162},"Agents","\u002Fbuild-plugins\u002Fagents","5.build-plugins\u002F3.agents","i-ph-robot",{"title":164,"path":165,"stem":166,"icon":167},"Skills","\u002Fbuild-plugins\u002Fskills","5.build-plugins\u002F4.skills","i-ph-graduation-cap",{"title":169,"path":170,"stem":171,"icon":172},"Develop and test","\u002Fbuild-plugins\u002Fdevelop-and-test","5.build-plugins\u002F5.develop-and-test","i-ph-flask",{"title":174,"path":175,"stem":176,"icon":177},"Publish and version","\u002Fbuild-plugins\u002Fpublish","5.build-plugins\u002F6.publish","i-ph-tag",{"title":179,"path":180,"stem":181,"icon":182},"Hooks","\u002Fbuild-plugins\u002Fhooks","5.build-plugins\u002F7.hooks","i-ph-anchor",{"title":184,"path":185,"stem":186,"icon":187},"Flows","\u002Fbuild-plugins\u002Fflows","5.build-plugins\u002F8.flows","i-ph-flow-arrow",{"title":189,"path":190,"stem":191,"icon":192},"Evals and quality gates","\u002Fbuild-plugins\u002Fevals","5.build-plugins\u002F9.evals","i-ph-seal-check",{"title":194,"icon":6,"path":195,"stem":196,"children":197,"page":6},"Run agents","\u002Fagents","6.agents",[198,203,208,213,218,223],{"title":199,"path":200,"stem":201,"icon":202},"Run jobs","\u002Fagents\u002Frun-jobs","6.agents\u002F1.run-jobs","i-ph-play",{"title":204,"path":205,"stem":206,"icon":207},"Models and keys","\u002Fagents\u002Fmodels-and-keys","6.agents\u002F2.models-and-keys","i-ph-sparkle",{"title":209,"path":210,"stem":211,"icon":212},"Delegation","\u002Fagents\u002Fdelegation","6.agents\u002F3.delegation","i-ph-git-fork",{"title":214,"path":215,"stem":216,"icon":217},"Contracts and limits","\u002Fagents\u002Fcontracts-and-limits","6.agents\u002F4.contracts-and-limits","i-ph-ruler",{"title":219,"path":220,"stem":221,"icon":222},"Runtime API","\u002Fagents\u002Fruntime-api","6.agents\u002F5.runtime-api","i-ph-plug",{"title":224,"path":225,"stem":226,"icon":227},"Approvals","\u002Fagents\u002Fapprovals","6.agents\u002F6.approvals","i-ph-hand-palm",{"title":229,"icon":6,"path":230,"stem":231,"children":232,"page":6},"Monitor","\u002Fmonitor","7.monitor",[233,238,243,248,253,258],{"title":234,"path":235,"stem":236,"icon":237},"Web UI","\u002Fmonitor\u002Fweb-ui","7.monitor\u002F0.web-ui","i-ph-monitor",{"title":239,"path":240,"stem":241,"icon":242},"Cost and usage","\u002Fmonitor\u002Fcost-and-usage","7.monitor\u002F1.cost-and-usage","i-ph-coins",{"title":244,"path":245,"stem":246,"icon":247},"Node health","\u002Fmonitor\u002Fnodes","7.monitor\u002F2.nodes","i-ph-heartbeat",{"title":249,"path":250,"stem":251,"icon":252},"Job history","\u002Fmonitor\u002Fhistory","7.monitor\u002F3.history","i-ph-clock-counter-clockwise",{"title":254,"path":255,"stem":256,"icon":257},"Tracing and metrics","\u002Fmonitor\u002Fobservability","7.monitor\u002F4.observability","i-ph-chart-line",{"title":259,"path":260,"stem":261,"icon":152},"Troubleshooting","\u002Fmonitor\u002Ftroubleshooting","7.monitor\u002F5.troubleshooting",{"title":263,"icon":6,"path":264,"stem":265,"children":266,"page":6},"Reference","\u002Freference","8.reference",[267,281,286,290,295,300],{"title":268,"icon":269,"path":270,"stem":271,"children":272,"page":6},"CLI","i-ph-terminal-window","\u002Freference\u002Fcli","8.reference\u002F1.cli",[273,277],{"title":274,"path":275,"stem":276},"orchestrator-zero","\u002Freference\u002Fcli\u002Forchestrator-zero","8.reference\u002F1.cli\u002F1.orchestrator-zero",{"title":278,"path":279,"stem":280},"orchestrator-zero-node","\u002Freference\u002Fcli\u002Forchestrator-zero-node","8.reference\u002F1.cli\u002F2.orchestrator-zero-node",{"title":282,"path":283,"stem":284,"icon":285},"oz0-plugin.yaml","\u002Freference\u002Fplugin-manifest","8.reference\u002F2.plugin-manifest","i-ph-file-code",{"title":287,"path":288,"stem":289,"icon":162},"Agent definition","\u002Freference\u002Fagent-definition","8.reference\u002F3.agent-definition",{"title":291,"path":292,"stem":293,"icon":294},"Ports","\u002Freference\u002Fports","8.reference\u002F4.ports","i-ph-network",{"title":296,"path":297,"stem":298,"icon":299},"Environment variables","\u002Freference\u002Fenvironment","8.reference\u002F5.environment","i-ph-brackets-curly",{"title":301,"path":302,"stem":303,"icon":304},"Files and directories","\u002Freference\u002Ffiles","8.reference\u002F6.files","i-ph-tree-structure",{"title":306,"icon":6,"path":307,"stem":308,"children":309,"page":6},"About","\u002Fabout","9.about",[310,315],{"title":311,"path":312,"stem":313,"icon":314},"Licensing","\u002Fabout\u002Flicensing","9.about\u002F1.licensing","i-ph-scales",{"title":316,"path":317,"stem":318,"icon":319},"About these docs","\u002Fabout\u002Fcontributing-docs","9.about\u002F2.contributing-docs","i-ph-pencil-simple",{"id":321,"title":105,"body":322,"description":1193,"extension":1194,"links":1195,"meta":1196,"navigation":1197,"path":106,"seo":1198,"stem":107,"__hash__":1199},"docs\u002F3.fleet\u002F7.upgrades.md",{"type":323,"value":324,"toc":1182},"minimark",[325,330,339,342,346,357,366,371,382,462,465,473,487,491,494,540,543,560,567,574,591,597,600,604,621,624,651,654,657,661,665,671,767,784,795,814,817,836,843,849,856,892,904,908,914,959,962,992,996,1085,1088,1092,1148,1151,1154,1171,1178],[326,327,329],"h2",{"id":328},"upgrade-servers","Upgrade servers",[331,332,333,334,338],"p",{},"Upgrade one server at a time. ",[335,336,337],"code",{},"server start"," applies pending schema migrations, ours and Temporal's, under a PostgreSQL advisory lock, so the first upgraded server migrates and the others wait. A binary refuses to start on a database whose schema is newer than it knows.",[331,340,341],{},"Temporal must move one minor version at a time. Read the release notes before you skip a release.",[326,343,345],{"id":344},"upgrade-nodes","Upgrade nodes",[331,347,348,349,352,353,356],{},"Every job stays on the runtime version it started on. Each tenant's Temporal namespace has a Worker Deployment, ",[335,350,351],{},"oz0-node",", and a runtime's version is its Build ID: the runtime's workers for agents, flows and gates are versioned, and jobs are pinned to the version that started them, so an upgrade never replays a job's history with other workflow code. The node's own queue, ",[335,354,355],{},"node.\u003Cid>",", which carries its tool calls, is not versioned.",[331,358,359,360,365],{},"The edge points each tenant's deployment at the version its nodes are meant to run: the version most of the tenant's online nodes are told to run, or run when told nothing, becomes current as soon as one of them runs it, and new jobs go there. While a ",[361,362,364],"a",{"href":363},"#roll-a-runtime-out-in-steps","rollout"," runs, it moves the deployment instead.",[367,368,370],"h3",{"id":369},"upgrade-the-runtime","Upgrade the runtime",[331,372,373,374,377,378,381],{},"The edge serves every runtime version in its dist directory: ",[335,375,376],{},"make dist"," adds the one you build to ",[335,379,380],{},"dist\u002Fruntime\u002F",", and the newest is what new nodes install. Tell nodes to run another one:",[383,384,390],"pre",{"className":385,"code":386,"filename":387,"language":388,"meta":389,"style":389},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","orchestrator-zero node runtimes               # the versions the edge serves, and how many nodes run each\norchestrator-zero node upgrade node-7f3a01b2c4d5 --runtime 0.5.0+1a2b3c4d5e6f\norchestrator-zero node upgrade --all-in default --runtime 0.5.0+1a2b3c4d5e6f\norchestrator-zero node list                   # RUNTIME: what each node runs, and what it moves to\n","Terminal","bash","",[335,391,392,411,430,449],{"__ignoreMap":389},[393,394,397,400,404,407],"span",{"class":395,"line":396},"line",1,[393,398,274],{"class":399},"sBMFI",[393,401,403],{"class":402},"sfazB"," node",[393,405,406],{"class":402}," runtimes",[393,408,410],{"class":409},"sHwdD","               # the versions the edge serves, and how many nodes run each\n",[393,412,414,416,418,421,424,427],{"class":395,"line":413},2,[393,415,274],{"class":399},[393,417,403],{"class":402},[393,419,420],{"class":402}," upgrade",[393,422,423],{"class":402}," node-7f3a01b2c4d5",[393,425,426],{"class":402}," --runtime",[393,428,429],{"class":402}," 0.5.0+1a2b3c4d5e6f\n",[393,431,433,435,437,439,442,445,447],{"class":395,"line":432},3,[393,434,274],{"class":399},[393,436,403],{"class":402},[393,438,420],{"class":402},[393,440,441],{"class":402}," --all-in",[393,443,444],{"class":402}," default",[393,446,426],{"class":402},[393,448,429],{"class":402},[393,450,452,454,456,459],{"class":395,"line":451},4,[393,453,274],{"class":399},[393,455,403],{"class":402},[393,457,458],{"class":402}," list",[393,460,461],{"class":409},"                   # RUNTIME: what each node runs, and what it moves to\n",[331,463,464],{},"A node told to run a version downloads it from the edge, checks its digest, installs it and starts it next to the one it runs. New jobs go to the new version once most of the tenant's nodes run it, and jobs that started on the old version finish there. When no job is pinned to the old version any more, which Temporal reports within half a minute of the last one ending, the nodes stop it:",[383,466,471],{"className":467,"code":469,"language":470,"meta":389},[468],"language-text","NODE               NAME    TENANT   STATE     STATUS         HEALTH       RUNTIME\nnode-40f20944d825  node-a  default  accepted  online on dev  healthy 100  0.5.0+1a2b3c4d5e6f, 0.4.0+9f8e7d6c5b4a\n","text",[335,472,469],{"__ignoreMap":389},[331,474,475,478,479,482,483,486],{},[335,476,477],{},"job get"," shows the runtime a job runs on. A version that cannot be downloaded or installed leaves the node on what it runs, and ",[335,480,481],{},"node show"," says why. ",[335,484,485],{},"node upgrade --runtime \"\""," lets nodes keep what they run.",[367,488,490],{"id":489},"upgrade-the-node-binary","Upgrade the node binary",[331,492,493],{},"Nodes replace their own binary with the one the edge serves:",[383,495,497],{"className":385,"code":496,"filename":387,"language":388,"meta":389,"style":389},"orchestrator-zero node upgrade node-7f3a01b2c4d5 --binary\norchestrator-zero node upgrade --all-in default --binary\norchestrator-zero node show node-7f3a01b2c4d5        # Supervisor: the version it runs\n",[335,498,499,512,526],{"__ignoreMap":389},[393,500,501,503,505,507,509],{"class":395,"line":396},[393,502,274],{"class":399},[393,504,403],{"class":402},[393,506,420],{"class":402},[393,508,423],{"class":402},[393,510,511],{"class":402}," --binary\n",[393,513,514,516,518,520,522,524],{"class":395,"line":413},[393,515,274],{"class":399},[393,517,403],{"class":402},[393,519,420],{"class":402},[393,521,441],{"class":402},[393,523,444],{"class":402},[393,525,511],{"class":402},[393,527,528,530,532,535,537],{"class":395,"line":432},[393,529,274],{"class":399},[393,531,403],{"class":402},[393,533,534],{"class":402}," show",[393,536,423],{"class":402},[393,538,539],{"class":409},"        # Supervisor: the version it runs\n",[331,541,542],{},"A node told to run another binary downloads the edge's release, and checks it before it changes anything:",[544,545,546,554,557],"ol",{},[547,548,549,550,553],"li",{},"The release's ",[335,551,552],{},"manifest.json"," must be signed with a key the node's own binary trusts.",[547,555,556],{},"The binary for its platform must match the digest in that manifest.",[547,558,559],{},"The new binary must start and say it is the version asked for.",[331,561,562,563,566],{},"Then the node keeps its old binary next to the new one as ",[335,564,565],{},"orchestrator-zero-node.previous",", stops its runtimes and starts again as the new binary, in the same process. A release the node does not trust leaves it on what it runs, and its log says why.",[331,568,569,570,573],{},"The new binary has two minutes to reach an edge (",[335,571,572],{},"OZ0_UPDATE_GRACE"," changes it). If it does not, or it stops before then, the node goes back:",[575,576,577,584],"ul",{},[547,578,579,580,583],{},"the previous binary is put back in place, and the new one is kept as ",[335,581,582],{},"orchestrator-zero-node.failed",";",[547,585,586,587,590],{},"the node does not install that version again. Upgrade it to another one, or delete ",[335,588,589],{},"node-update.json"," from the node's data directory and restart the node to try again.",[331,592,593,594,596],{},"The node records the update in ",[335,595,589],{}," before the new binary starts. A new binary that crashes, and is started again by a service manager, still finds it and goes back.",[331,598,599],{},"Runtimes are checked the same way: a node takes a runtime's digest only from the signed manifest. A node that starts while no edge offers a runtime it trusts, because the edge is down or serves a release the node does not trust, starts the runtime it already has.",[367,601,603],{"id":602},"releases-and-their-signatures","Releases and their signatures",[331,605,606,608,609,612,613,616,617,620],{},[335,607,376],{}," signs ",[335,610,611],{},"dist\u002Fmanifest.json"," with a release key, and every binary it builds trusts that key. The first time, it makes a key of its own in ",[335,614,615],{},"~\u002F.config\u002Forchestrator-zero\u002Frelease.key",". Keep that file safe: whoever has it can sign node binaries your nodes will run. Set ",[335,618,619],{},"RELEASE_KEY"," to keep it somewhere else.",[331,622,623],{},"Mirror a release that is served elsewhere, such as a release's download folder, into the edge's dist directory:",[383,625,627],{"className":385,"code":626,"filename":387,"language":388,"meta":389,"style":389},"orchestrator-zero server dist pull https:\u002F\u002Freleases.example.com\u002Forchestrator-zero\u002F0.7.0 --dist-dir \u002Fsrv\u002Foz0\u002Fdist\n",[335,628,629],{"__ignoreMap":389},[393,630,631,633,636,639,642,645,648],{"class":395,"line":396},[393,632,274],{"class":399},[393,634,635],{"class":402}," server",[393,637,638],{"class":402}," dist",[393,640,641],{"class":402}," pull",[393,643,644],{"class":402}," https:\u002F\u002Freleases.example.com\u002Forchestrator-zero\u002F0.7.0",[393,646,647],{"class":402}," --dist-dir",[393,649,650],{"class":402}," \u002Fsrv\u002Foz0\u002Fdist\n",[331,652,653],{},"The server checks the release's signature against the keys its own binary trusts, downloads every file it names and checks each digest. The manifest goes in place last, so nodes see the new release whole or not at all. Node binaries of releases older than the one before are removed from the dist directory; runtimes stay until you delete them.",[331,655,656],{},"A signature proves that a release is yours, not that it is the newest: someone who controls an edge can serve an older release you signed. To retire a release for good, sign the next ones with a new key and upgrade the nodes to binaries that trust only it.",[658,659,660],"note",{},"The one-line install trusts the edge: it pins the cluster's CA and checks the binary's digest from the edge's manifest. From then on, the node checks signatures itself.",[367,662,664],{"id":663},"roll-a-runtime-out-in-steps","Roll a runtime out in steps",[331,666,667,670],{},[335,668,669],{},"node upgrade"," moves the nodes you name at once. A rollout moves all of a tenant's nodes in steps, and stops by itself when the new version does worse:",[383,672,674],{"className":385,"code":673,"filename":387,"language":388,"meta":389,"style":389},"orchestrator-zero rollout start --runtime 0.5.0+1a2b3c4d5e6f       # tenant default: a canary, then the rest in two steps\norchestrator-zero rollout start --runtime 0.5.0+1a2b3c4d5e6f --tenant acme --canary 2 --batch 5 --soak 10m\norchestrator-zero rollout show rollout-3f9c2a1b\norchestrator-zero rollout list\norchestrator-zero rollout abort rollout-3f9c2a1b                   # roll back by hand\n",[335,675,676,694,731,742,751],{"__ignoreMap":389},[393,677,678,680,683,686,688,691],{"class":395,"line":396},[393,679,274],{"class":399},[393,681,682],{"class":402}," rollout",[393,684,685],{"class":402}," start",[393,687,426],{"class":402},[393,689,690],{"class":402}," 0.5.0+1a2b3c4d5e6f",[393,692,693],{"class":409},"       # tenant default: a canary, then the rest in two steps\n",[393,695,696,698,700,702,704,706,709,712,715,719,722,725,728],{"class":395,"line":413},[393,697,274],{"class":399},[393,699,682],{"class":402},[393,701,685],{"class":402},[393,703,426],{"class":402},[393,705,690],{"class":402},[393,707,708],{"class":402}," --tenant",[393,710,711],{"class":402}," acme",[393,713,714],{"class":402}," --canary",[393,716,718],{"class":717},"sbssI"," 2",[393,720,721],{"class":402}," --batch",[393,723,724],{"class":717}," 5",[393,726,727],{"class":402}," --soak",[393,729,730],{"class":402}," 10m\n",[393,732,733,735,737,739],{"class":395,"line":432},[393,734,274],{"class":399},[393,736,682],{"class":402},[393,738,534],{"class":402},[393,740,741],{"class":402}," rollout-3f9c2a1b\n",[393,743,744,746,748],{"class":395,"line":451},[393,745,274],{"class":399},[393,747,682],{"class":402},[393,749,750],{"class":402}," list\n",[393,752,754,756,758,761,764],{"class":395,"line":753},5,[393,755,274],{"class":399},[393,757,682],{"class":402},[393,759,760],{"class":402}," abort",[393,762,763],{"class":402}," rollout-3f9c2a1b",[393,765,766],{"class":409},"                   # roll back by hand\n",[331,768,769,770,774,775,779,780,783],{},"In the web UI, ",[771,772,773],"strong",{},"Upgrade"," on the ",[776,777,778],"em",{},"Nodes"," page starts one: choose a runtime the edge serves, the canary, the nodes per later step, the soak and the start timeout. The panel above the nodes follows each step to its verdict, and ",[771,781,782],{},"Roll back"," aborts it.",[331,785,786,787,790,791,794],{},"A rollout plans every online node of the tenant that does not already run only the new version: a canary of one node (",[335,788,789],{},"--canary","), then the others in steps of half of them each (",[335,792,793],{},"--batch","). Each step:",[544,796,797,804,807],{},[547,798,799,800,803],{},"tells its nodes to run the new version, and waits until they run it, for at most ",[335,801,802],{},"--start-timeout"," (5 minutes);",[547,805,806],{},"sends the new version its share of the tenant's new jobs: with one node of four upgraded, a quarter of them;",[547,808,809,810,813],{},"watches the upgraded nodes for ",[335,811,812],{},"--soak"," (1 minute), and then judges the step.",[331,815,816],{},"A step passes two gates:",[575,818,819,830],{},[547,820,821,824,825,829],{},[771,822,823],{},"Health:"," every upgraded node stays online, keeps running the new version and stays ",[361,826,828],{"href":827},"\u002Fmonitor\u002Fnodes#health-scores","healthy",", all through the soak. A node that goes bad fails the step at once.",[547,831,832,835],{},[771,833,834],{},"Quality:"," of the jobs on the new version that ended during the soak, the share that failed is at most 10 points worse than on the old version over the same time. The gate needs five such jobs; a quiet tenant passes on health alone, and the step says so.",[331,837,838,839,842],{},"When the last step passes, the new version becomes current and the old one drains, as above. When a step fails, or you abort, the rollout rolls back by itself: the upgraded nodes are told to run the old version again, the old version becomes current again, and jobs that started on the new version finish there. A runtime that does not start never gets past its canary. Here the canary's new runtime exited as soon as it started, with ",[335,840,841],{},"--start-timeout 20s",":",[383,844,847],{"className":845,"code":846,"language":470,"meta":389},[468],"Rollout rollout-0ca52a09: tenant default, runtime from 0.5.0+1a2b3c4d5e6f to 0.5.1+7c6d5e4f3a2b, rolled-back\n  canary  failed   node-07ecbf417308: step 1: node node-07ecbf417308 did not run 0.5.1+7c6d5e4f3a2b (RUNTIME_STATE_CRASHED after 4 restarts: exited with status 0) within 20s\n  step 2  pending  node-5cd237748ea5\n  step 3  pending  node-e8db051fed0d\nReason: step 1: node node-07ecbf417308 did not run 0.5.1+7c6d5e4f3a2b (RUNTIME_STATE_CRASHED after 4 restarts: exited with status 0) within 20s\n",[335,848,846],{"__ignoreMap":389},[331,850,851,852,855],{},"In ",[335,853,854],{},"make e2e-rollout",", which does this while jobs run back to back, none of the 39 jobs that ran meanwhile failed: the other nodes, and the canary's old runtime, took them.",[331,857,858,859,862,863,866,867,870,871,870,874,870,877,870,880,883,884,887,888,891],{},"A tenant has one rollout at a time: ",[335,860,861],{},"rollout start"," refuses while another runs. A server with the edge role drives it and holds a lease on it in the database; if that server stops, another one carries on from the same step within half a minute. The audit log has every rollout (",[335,864,865],{},"orchestrator-zero audit --action rollout.",": ",[335,868,869],{},"create",", ",[335,872,873],{},"start",[335,875,876],{},"step",[335,878,879],{},"done",[335,881,882],{},"rolled-back"," and ",[335,885,886],{},"abort","), and the metric ",[335,889,890],{},"oz0_rollouts_total"," counts how they ended.",[331,893,894,895,899,900,903],{},"Plugin versions roll out the same way, canary first, when a plugin that runs on two or more nodes gets a new version: see ",[361,896,898],{"href":897},"\u002Fplugins\u002Finstall#new-versions-roll-out-to-canaries-first","New versions roll out to canaries first",". Such a rollout waits while a runtime rollout runs, and ",[335,901,902],{},"rollout list"," shows both kinds.",[326,905,907],{"id":906},"back-up","Back up",[331,909,910,913],{},[335,911,912],{},"server backup"," writes one archive of the management database while the servers keep running. It reads every table in one snapshot, and for a dev server it copies the embedded Temporal's database too:",[383,915,917],{"className":385,"code":916,"filename":387,"language":388,"meta":389,"style":389},"orchestrator-zero server backup --out oz0-backup.tar.gz                          # a cluster: --db or OZ0_DB\norchestrator-zero server backup --dev --data-dir .oz0\u002Fdev --out oz0-backup.tar.gz\n",[335,918,919,937],{"__ignoreMap":389},[393,920,921,923,925,928,931,934],{"class":395,"line":396},[393,922,274],{"class":399},[393,924,635],{"class":402},[393,926,927],{"class":402}," backup",[393,929,930],{"class":402}," --out",[393,932,933],{"class":402}," oz0-backup.tar.gz",[393,935,936],{"class":409},"                          # a cluster: --db or OZ0_DB\n",[393,938,939,941,943,945,948,951,954,956],{"class":395,"line":413},[393,940,274],{"class":399},[393,942,635],{"class":402},[393,944,927],{"class":402},[393,946,947],{"class":402}," --dev",[393,949,950],{"class":402}," --data-dir",[393,952,953],{"class":402}," .oz0\u002Fdev",[393,955,930],{"class":402},[393,957,958],{"class":402}," oz0-backup.tar.gz\n",[331,960,961],{},"The archive holds tenants, nodes and join tokens, plugins and their artifacts, sealed secrets, accounts and memberships, usage, the audit log, job summaries, and the large payloads that jobs keep out of their histories. It leaves out:",[575,963,964,973,979],{},[547,965,966,969,970,583],{},[771,967,968],{},"the master key."," Keep it apart, and give it to ",[335,971,972],{},"server restore",[547,974,975,978],{},[771,976,977],{},"what describes the moment:"," live servers, nodes' connections and sign-in sessions;",[547,980,981,984,985,987,988,991],{},[771,982,983],{},"the dist directory,"," which ",[335,986,376],{}," or ",[335,989,990],{},"server dist pull"," fills again.",[993,994,995],"warning",{},"The archive holds what jobs read and wrote, and every secret, sealed. Store it as carefully as the database. Nobody can restore it, or read its secrets, without the master key.",[997,998,999,1015],"table",{},[1000,1001,1002],"thead",{},[1003,1004,1005,1009,1012],"tr",{},[1006,1007,1008],"th",{},"What",[1006,1010,1011],{},"Why",[1006,1013,1014],{},"How",[1016,1017,1018,1035,1050,1064],"tbody",{},[1003,1019,1020,1026,1029],{},[1021,1022,1023],"td",{},[771,1024,1025],{},"The master key file",[1021,1027,1028],{},"Without it, the CAs and secrets cannot be unsealed and the cluster is lost",[1021,1030,1031,1032],{},"Copy it somewhere safe and offline once, after ",[335,1033,1034],{},"server init",[1003,1036,1037,1042,1045],{},[1021,1038,1039],{},[771,1040,1041],{},"The management database",[1021,1043,1044],{},"Everything in the list above",[1021,1046,1047,1049],{},[335,1048,912],{}," on a schedule, or PostgreSQL's own backups for point-in-time recovery",[1003,1051,1052,1058,1061],{},[1021,1053,1054,1057],{},[771,1055,1056],{},"Temporal's two databases"," (a cluster)",[1021,1059,1060],{},"Running jobs and their recent histories",[1021,1062,1063],{},"PostgreSQL's own backups: the archive does not hold them",[1003,1065,1066,1071,1074],{},[1021,1067,1068],{},[771,1069,1070],{},"Operator contexts",[1021,1072,1073],{},"Operator certificates",[1021,1075,1076,1077,1080,1081,1084],{},"Renew them with ",[335,1078,1079],{},"operator renew",". If one is lost or expired, issue new credentials on a server with ",[335,1082,1083],{},"server operator issue",". Keep them private",[331,1086,1087],{},"Nodes hold nothing you need to back up: their plugins come from the edge, and a lost node can simply join again.",[326,1089,1091],{"id":1090},"restore","Restore",[383,1093,1095],{"className":385,"code":1094,"filename":387,"language":388,"meta":389,"style":389},"orchestrator-zero server restore oz0-backup.tar.gz --master-key-file \u002Fetc\u002Forchestrator-zero\u002Fmaster.key \\\n  --db postgres:\u002F\u002Foz0@db.example.com:5432\u002Foz0\norchestrator-zero server restore oz0-backup.tar.gz --master-key-file .oz0\u002Fdev\u002Fmaster.key --dev --data-dir .oz0\u002Frestored\n",[335,1096,1097,1118,1126],{"__ignoreMap":389},[393,1098,1099,1101,1103,1106,1108,1111,1114],{"class":395,"line":396},[393,1100,274],{"class":399},[393,1102,635],{"class":402},[393,1104,1105],{"class":402}," restore",[393,1107,933],{"class":402},[393,1109,1110],{"class":402}," --master-key-file",[393,1112,1113],{"class":402}," \u002Fetc\u002Forchestrator-zero\u002Fmaster.key",[393,1115,1117],{"class":1116},"sTEyZ"," \\\n",[393,1119,1120,1123],{"class":395,"line":413},[393,1121,1122],{"class":402},"  --db",[393,1124,1125],{"class":402}," postgres:\u002F\u002Foz0@db.example.com:5432\u002Foz0\n",[393,1127,1128,1130,1132,1134,1136,1138,1141,1143,1145],{"class":395,"line":432},[393,1129,274],{"class":399},[393,1131,635],{"class":402},[393,1133,1105],{"class":402},[393,1135,933],{"class":402},[393,1137,1110],{"class":402},[393,1139,1140],{"class":402}," .oz0\u002Fdev\u002Fmaster.key",[393,1142,947],{"class":402},[393,1144,950],{"class":402},[393,1146,1147],{"class":402}," .oz0\u002Frestored\n",[331,1149,1150],{},"A restore goes into an empty database: a new PostgreSQL database, or a new dev data directory. Before it writes anything, it checks every file in the archive against its digest, and checks that the master key unseals the backup's CAs. Then it inserts every row in one transaction and migrates to the binary's schema. A restore that fails leaves a dev data directory as it found it.",[331,1152,1153],{},"After a restore:",[575,1155,1156,1159,1162,1168],{},[547,1157,1158],{},"start the servers with the same master key;",[547,1160,1161],{},"nodes keep their certificates and reconnect when they reach the edge at the address they know;",[547,1163,1164,1165,1167],{},"operators get new credentials with ",[335,1166,1083],{},". A dev server issues its own when it starts;",[547,1169,1170],{},"accounts keep their passwords, and sign in again.",[331,1172,1173,1174,1177],{},"A cluster restored without Temporal's databases has everything but the jobs that were running and their recent histories. A dev backup has those too. ",[335,1175,1176],{},"make e2e-backup"," backs up a dev server while a job waits for a person, restores it into a new data directory, and the job finishes there once it is approved.",[1179,1180,1181],"style",{},"html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sbssI, html code.shiki .sbssI{--shiki-light:#F76D47;--shiki-default:#F78C6C;--shiki-dark:#F78C6C}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}",{"title":389,"searchDepth":413,"depth":413,"links":1183},[1184,1185,1191,1192],{"id":328,"depth":413,"text":329},{"id":344,"depth":413,"text":345,"children":1186},[1187,1188,1189,1190],{"id":369,"depth":432,"text":370},{"id":489,"depth":432,"text":490},{"id":602,"depth":432,"text":603},{"id":663,"depth":432,"text":664},{"id":906,"depth":413,"text":907},{"id":1090,"depth":413,"text":1091},"Upgrade servers and nodes safely, back up what you cannot lose, and restore it.","md",null,{},{"icon":108},{"title":105,"description":1193},"SQk9L3jUFci-eZMMtes13fP3-_2qcEF9mRHpJSv4qVA",[1201,1203],{"title":100,"path":101,"stem":102,"description":1202,"icon":103,"children":-1},"Run two or more servers so nodes and jobs keep going when one stops.",{"title":115,"path":116,"stem":117,"description":1204,"icon":118,"children":-1},"Everything specific to your company ships as a plugin, a Git repository with one manifest.",1791135408128]