[{"data":1,"prerenderedAt":756},["ShallowReactive",2],{"navigation_docs":3,"-monitor-nodes":320,"-monitor-nodes-surround":751},[4,35,69,109,133,193,228,262,305],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":6},"Getting started",false,"\u002Fgetting-started","1.getting-started",[10,15,20,25,30],{"title":11,"path":12,"stem":13,"icon":14},"Introduction","\u002Fgetting-started\u002Fintroduction","1.getting-started\u002F1.introduction","i-ph-house",{"title":16,"path":17,"stem":18,"icon":19},"Quickstart","\u002Fgetting-started\u002Fquickstart","1.getting-started\u002F2.quickstart","i-ph-rocket-launch",{"title":21,"path":22,"stem":23,"icon":24},"Concepts","\u002Fgetting-started\u002Fconcepts","1.getting-started\u002F3.concepts","i-ph-book-open",{"title":26,"path":27,"stem":28,"icon":29},"Install","\u002Fgetting-started\u002Finstall","1.getting-started\u002F4.install","i-ph-download-simple",{"title":31,"path":32,"stem":33,"icon":34},"What works today","\u002Fgetting-started\u002Fstatus","1.getting-started\u002F5.status","i-ph-list-checks",{"title":21,"icon":6,"path":36,"stem":37,"children":38,"page":6},"\u002Fconcepts","2.concepts",[39,44,49,54,59,64],{"title":40,"path":41,"stem":42,"icon":43},"Architecture","\u002Fconcepts\u002Farchitecture","2.concepts\u002F1.architecture","i-ph-planet",{"title":45,"path":46,"stem":47,"icon":48},"Jobs and durability","\u002Fconcepts\u002Fjobs","2.concepts\u002F2.jobs","i-ph-infinity",{"title":50,"path":51,"stem":52,"icon":53},"Routing","\u002Fconcepts\u002Frouting","2.concepts\u002F3.routing","i-ph-path",{"title":55,"path":56,"stem":57,"icon":58},"Security model","\u002Fconcepts\u002Fsecurity","2.concepts\u002F4.security","i-ph-shield-check",{"title":60,"path":61,"stem":62,"icon":63},"Tenants","\u002Fconcepts\u002Ftenants","2.concepts\u002F5.tenants","i-ph-buildings",{"title":65,"path":66,"stem":67,"icon":68},"Temporal","\u002Fconcepts\u002Ftemporal","2.concepts\u002F6.temporal","i-ph-clock",{"title":70,"icon":6,"path":71,"stem":72,"children":73,"page":6},"Run a fleet","\u002Ffleet","3.fleet",[74,79,84,89,94,99,104],{"title":75,"path":76,"stem":77,"icon":78},"Dev mode","\u002Ffleet\u002Fdev-mode","3.fleet\u002F1.dev-mode","i-ph-laptop",{"title":80,"path":81,"stem":82,"icon":83},"Run a cluster","\u002Ffleet\u002Fcluster","3.fleet\u002F2.cluster","i-ph-hard-drives",{"title":85,"path":86,"stem":87,"icon":88},"Join nodes","\u002Ffleet\u002Fjoin-nodes","3.fleet\u002F3.join-nodes","i-ph-plugs-connected",{"title":90,"path":91,"stem":92,"icon":93},"Manage nodes","\u002Ffleet\u002Fmanage-nodes","3.fleet\u002F4.manage-nodes","i-ph-list-bullets",{"title":95,"path":96,"stem":97,"icon":98},"Secrets","\u002Ffleet\u002Fsecrets","3.fleet\u002F5.secrets","i-ph-key",{"title":100,"path":101,"stem":102,"icon":103},"High availability","\u002Ffleet\u002Fhigh-availability","3.fleet\u002F6.high-availability","i-ph-copy",{"title":105,"path":106,"stem":107,"icon":108},"Upgrades, backups and restores","\u002Ffleet\u002Fupgrades","3.fleet\u002F7.upgrades","i-ph-arrows-clockwise",{"title":110,"icon":6,"path":111,"stem":112,"children":113,"page":6},"Use plugins","\u002Fplugins","4.plugins",[114,119,124,128],{"title":115,"path":116,"stem":117,"icon":118},"What a plugin is","\u002Fplugins\u002Foverview","4.plugins\u002F1.overview","i-ph-puzzle-piece",{"title":120,"path":121,"stem":122,"icon":123},"Install and update","\u002Fplugins\u002Finstall","4.plugins\u002F2.install","i-ph-package",{"title":125,"path":126,"stem":127,"icon":123},"Existing MCP packages","\u002Fplugins\u002Fpackages","4.plugins\u002F3.packages",{"title":129,"path":130,"stem":131,"icon":132},"The catalog","\u002Fplugins\u002Fcatalog","4.plugins\u002F4.catalog","i-ph-books",{"title":134,"icon":6,"path":135,"stem":136,"children":137,"page":6},"Build plugins","\u002Fbuild-plugins","5.build-plugins",[138,143,148,153,158,163,168,173,178,183,188],{"title":139,"path":140,"stem":141,"icon":142},"Your first plugin","\u002Fbuild-plugins\u002Ffirst-plugin","5.build-plugins\u002F1.first-plugin","i-ph-hammer",{"title":144,"path":145,"stem":146,"icon":147},"Harnesses","\u002Fbuild-plugins\u002Fharnesses","5.build-plugins\u002F10.harnesses","i-ph-cpu",{"title":149,"path":150,"stem":151,"icon":152},"Example: support triage","\u002Fbuild-plugins\u002Fexample-support-triage","5.build-plugins\u002F11.example-support-triage","i-ph-lifebuoy",{"title":154,"path":155,"stem":156,"icon":157},"Tools","\u002Fbuild-plugins\u002Ftools","5.build-plugins\u002F2.tools","i-ph-wrench",{"title":159,"path":160,"stem":161,"icon":162},"Agents","\u002Fbuild-plugins\u002Fagents","5.build-plugins\u002F3.agents","i-ph-robot",{"title":164,"path":165,"stem":166,"icon":167},"Skills","\u002Fbuild-plugins\u002Fskills","5.build-plugins\u002F4.skills","i-ph-graduation-cap",{"title":169,"path":170,"stem":171,"icon":172},"Develop and test","\u002Fbuild-plugins\u002Fdevelop-and-test","5.build-plugins\u002F5.develop-and-test","i-ph-flask",{"title":174,"path":175,"stem":176,"icon":177},"Publish and version","\u002Fbuild-plugins\u002Fpublish","5.build-plugins\u002F6.publish","i-ph-tag",{"title":179,"path":180,"stem":181,"icon":182},"Hooks","\u002Fbuild-plugins\u002Fhooks","5.build-plugins\u002F7.hooks","i-ph-anchor",{"title":184,"path":185,"stem":186,"icon":187},"Flows","\u002Fbuild-plugins\u002Fflows","5.build-plugins\u002F8.flows","i-ph-flow-arrow",{"title":189,"path":190,"stem":191,"icon":192},"Evals and quality gates","\u002Fbuild-plugins\u002Fevals","5.build-plugins\u002F9.evals","i-ph-seal-check",{"title":194,"icon":6,"path":195,"stem":196,"children":197,"page":6},"Run agents","\u002Fagents","6.agents",[198,203,208,213,218,223],{"title":199,"path":200,"stem":201,"icon":202},"Run jobs","\u002Fagents\u002Frun-jobs","6.agents\u002F1.run-jobs","i-ph-play",{"title":204,"path":205,"stem":206,"icon":207},"Models and keys","\u002Fagents\u002Fmodels-and-keys","6.agents\u002F2.models-and-keys","i-ph-sparkle",{"title":209,"path":210,"stem":211,"icon":212},"Delegation","\u002Fagents\u002Fdelegation","6.agents\u002F3.delegation","i-ph-git-fork",{"title":214,"path":215,"stem":216,"icon":217},"Contracts and limits","\u002Fagents\u002Fcontracts-and-limits","6.agents\u002F4.contracts-and-limits","i-ph-ruler",{"title":219,"path":220,"stem":221,"icon":222},"Runtime API","\u002Fagents\u002Fruntime-api","6.agents\u002F5.runtime-api","i-ph-plug",{"title":224,"path":225,"stem":226,"icon":227},"Approvals","\u002Fagents\u002Fapprovals","6.agents\u002F6.approvals","i-ph-hand-palm",{"title":229,"icon":6,"path":230,"stem":231,"children":232,"page":6},"Monitor","\u002Fmonitor","7.monitor",[233,238,243,248,253,258],{"title":234,"path":235,"stem":236,"icon":237},"Web UI","\u002Fmonitor\u002Fweb-ui","7.monitor\u002F0.web-ui","i-ph-monitor",{"title":239,"path":240,"stem":241,"icon":242},"Cost and usage","\u002Fmonitor\u002Fcost-and-usage","7.monitor\u002F1.cost-and-usage","i-ph-coins",{"title":244,"path":245,"stem":246,"icon":247},"Node health","\u002Fmonitor\u002Fnodes","7.monitor\u002F2.nodes","i-ph-heartbeat",{"title":249,"path":250,"stem":251,"icon":252},"Job history","\u002Fmonitor\u002Fhistory","7.monitor\u002F3.history","i-ph-clock-counter-clockwise",{"title":254,"path":255,"stem":256,"icon":257},"Tracing and metrics","\u002Fmonitor\u002Fobservability","7.monitor\u002F4.observability","i-ph-chart-line",{"title":259,"path":260,"stem":261,"icon":152},"Troubleshooting","\u002Fmonitor\u002Ftroubleshooting","7.monitor\u002F5.troubleshooting",{"title":263,"icon":6,"path":264,"stem":265,"children":266,"page":6},"Reference","\u002Freference","8.reference",[267,281,286,290,295,300],{"title":268,"icon":269,"path":270,"stem":271,"children":272,"page":6},"CLI","i-ph-terminal-window","\u002Freference\u002Fcli","8.reference\u002F1.cli",[273,277],{"title":274,"path":275,"stem":276},"orchestrator-zero","\u002Freference\u002Fcli\u002Forchestrator-zero","8.reference\u002F1.cli\u002F1.orchestrator-zero",{"title":278,"path":279,"stem":280},"orchestrator-zero-node","\u002Freference\u002Fcli\u002Forchestrator-zero-node","8.reference\u002F1.cli\u002F2.orchestrator-zero-node",{"title":282,"path":283,"stem":284,"icon":285},"oz0-plugin.yaml","\u002Freference\u002Fplugin-manifest","8.reference\u002F2.plugin-manifest","i-ph-file-code",{"title":287,"path":288,"stem":289,"icon":162},"Agent definition","\u002Freference\u002Fagent-definition","8.reference\u002F3.agent-definition",{"title":291,"path":292,"stem":293,"icon":294},"Ports","\u002Freference\u002Fports","8.reference\u002F4.ports","i-ph-network",{"title":296,"path":297,"stem":298,"icon":299},"Environment variables","\u002Freference\u002Fenvironment","8.reference\u002F5.environment","i-ph-brackets-curly",{"title":301,"path":302,"stem":303,"icon":304},"Files and directories","\u002Freference\u002Ffiles","8.reference\u002F6.files","i-ph-tree-structure",{"title":306,"icon":6,"path":307,"stem":308,"children":309,"page":6},"About","\u002Fabout","9.about",[310,315],{"title":311,"path":312,"stem":313,"icon":314},"Licensing","\u002Fabout\u002Flicensing","9.about\u002F1.licensing","i-ph-scales",{"title":316,"path":317,"stem":318,"icon":319},"About these docs","\u002Fabout\u002Fcontributing-docs","9.about\u002F2.contributing-docs","i-ph-pencil-simple",{"id":321,"title":244,"body":322,"description":744,"extension":745,"links":746,"meta":747,"navigation":748,"path":245,"seo":749,"stem":246,"__hash__":750},"docs\u002F7.monitor\u002F2.nodes.md",{"type":323,"value":324,"toc":739},"minimark",[325,330,425,429,433,436,444,522,590,593,644,650,662,669,673,676,713,720,727,735],[326,327,329],"h2",{"id":328},"what-you-can-see-today","What you can see today",[331,332,338],"pre",{"className":333,"code":334,"filename":335,"language":336,"meta":337,"style":337},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","orchestrator-zero node list                   # every node with its state\norchestrator-zero node show \u003Cnode-id>         # one node in detail\norchestrator-zero plugin list                 # each node's state with each plugin\norchestrator-zero catalog                     # which nodes are ready to run each agent\norchestrator-zero cluster info                # the servers that are up\n","Terminal","bash","",[339,340,341,360,387,400,411],"code",{"__ignoreMap":337},[342,343,346,349,353,356],"span",{"class":344,"line":345},"line",1,[342,347,274],{"class":348},"sBMFI",[342,350,352],{"class":351},"sfazB"," node",[342,354,355],{"class":351}," list",[342,357,359],{"class":358},"sHwdD","                   # every node with its state\n",[342,361,363,365,367,370,374,377,381,384],{"class":344,"line":362},2,[342,364,274],{"class":348},[342,366,352],{"class":351},[342,368,369],{"class":351}," show",[342,371,373],{"class":372},"sMK4o"," \u003C",[342,375,376],{"class":351},"node-i",[342,378,380],{"class":379},"sTEyZ","d",[342,382,383],{"class":372},">",[342,385,386],{"class":358},"         # one node in detail\n",[342,388,390,392,395,397],{"class":344,"line":389},3,[342,391,274],{"class":348},[342,393,394],{"class":351}," plugin",[342,396,355],{"class":351},[342,398,399],{"class":358},"                 # each node's state with each plugin\n",[342,401,403,405,408],{"class":344,"line":402},4,[342,404,274],{"class":348},[342,406,407],{"class":351}," catalog",[342,409,410],{"class":358},"                     # which nodes are ready to run each agent\n",[342,412,414,416,419,422],{"class":344,"line":413},5,[342,415,274],{"class":348},[342,417,418],{"class":351}," cluster",[342,420,421],{"class":351}," info",[342,423,424],{"class":358},"                # the servers that are up\n",[426,427,428],"p",{},"Nodes keep a control stream open to the edge and send heartbeats over it. A node that is blocked, rejected or deleted has its stream closed within two seconds.",[326,430,432],{"id":431},"health-scores","Health scores",[426,434,435],{},"The edge that holds a node's control stream scores the node at every heartbeat, every ten seconds, from 0 to 100:",[331,437,442],{"className":438,"code":440,"language":441,"meta":337},[439],"language-text","score = 100 − resources − failures − latency − model calls − restarts\n","text",[339,443,440],{"__ignoreMap":337},[445,446,447,463],"table",{},[448,449,450],"thead",{},[451,452,453,457,460],"tr",{},[454,455,456],"th",{},"Penalty",[454,458,459],{},"Up to",[454,461,462],{},"Grows with",[464,465,466,478,489,500,511],"tbody",{},[451,467,468,472,475],{},[469,470,471],"td",{},"Resources",[469,473,474],{},"25",[469,476,477],{},"The busiest of CPU, memory, disk and GPU memory over 80%, smoothed over a few heartbeats",[451,479,480,483,486],{},[469,481,482],{},"Failures",[469,484,485],{},"30",[469,487,488],{},"The share of the node's activities in the last 15 minutes that failed because of the node or a plugin, over 2%. A tool that answers with an error, a model that is overloaded or a job out of budget is not the node's failure.",[451,490,491,494,497],{},[469,492,493],{},"Latency",[469,495,496],{},"15",[469,498,499],{},"How much slower the node runs a type of activity than the cluster's median, from 1.5 times; at least 20 runs",[451,501,502,505,508],{},[469,503,504],{},"Model calls",[469,506,507],{},"10",[469,509,510],{},"The share of its model calls in 15 minutes that got 429 or 5xx, or never reached the provider, over 5%",[451,512,513,516,519],{},[469,514,515],{},"Restarts",[469,517,518],{},"20",[469,520,521],{},"Restarts of its runtime in 15 minutes; three cost everything",[445,523,524,537],{},[448,525,526],{},[451,527,528,531,534],{},[454,529,530],{},"State",[454,532,533],{},"Score",[454,535,536],{},"What happens",[464,538,539,550,565,579],{},[451,540,541,544,547],{},[469,542,543],{},"Healthy",[469,545,546],{},"80–100",[469,548,549],{},"Nothing",[451,551,552,555,558],{},[469,553,554],{},"Degraded",[469,556,557],{},"50–79",[469,559,560,561,564],{},"The node takes half as many activities at once until it recovers. The audit log records ",[339,562,563],{},"node.degraded",".",[451,566,567,570,573],{},[469,568,569],{},"Unhealthy",[469,571,572],{},"0–49",[469,574,575,576,564],{},"The node gets no new tasks and drains: its polls come back empty, and what it runs finishes as usual. The audit log records ",[339,577,578],{},"node.unhealthy",[451,580,581,584,587],{},[469,582,583],{},"Offline",[469,585,586],{},"—",[469,588,589],{},"No heartbeat for 30 seconds. Its activities time out and run elsewhere.",[426,591,592],{},"A disk over 95% full, or a runtime that has been down for 30 seconds, makes a node unhealthy whatever its score. A state changes after two heartbeats in a row agree, and back to healthy takes three.",[331,594,596],{"className":333,"code":595,"filename":335,"language":336,"meta":337,"style":337},"orchestrator-zero node list                   # HEALTH: state and score\norchestrator-zero node show \u003Cnode-id>         # the reason and every penalty\norchestrator-zero audit --action node.        # when nodes changed state, and why\n",[339,597,598,609,628],{"__ignoreMap":337},[342,599,600,602,604,606],{"class":344,"line":345},[342,601,274],{"class":348},[342,603,352],{"class":351},[342,605,355],{"class":351},[342,607,608],{"class":358},"                   # HEALTH: state and score\n",[342,610,611,613,615,617,619,621,623,625],{"class":344,"line":362},[342,612,274],{"class":348},[342,614,352],{"class":351},[342,616,369],{"class":351},[342,618,373],{"class":372},[342,620,376],{"class":351},[342,622,380],{"class":379},[342,624,383],{"class":372},[342,626,627],{"class":358},"         # the reason and every penalty\n",[342,629,630,632,635,638,641],{"class":344,"line":389},[342,631,274],{"class":348},[342,633,634],{"class":351}," audit",[342,636,637],{"class":351}," --action",[342,639,640],{"class":351}," node.",[342,642,643],{"class":358},"        # when nodes changed state, and why\n",[331,645,648],{"className":646,"code":647,"language":441,"meta":337},[439],"Health      unhealthy 75: disk 97% full\nPenalties   resources 25.0, failures 0.0, latency 0.0, model calls 0.0, restarts 0.0 (scored 3s ago)\n",[339,649,647],{"__ignoreMap":337},[426,651,652,653,657,658,661],{},"The web UI's ",[654,655,656],"em",{},"Nodes"," page shows the same, with the penalties on the badge's tooltip. A tool with ",[339,659,660],{},"affinity: node"," runs only on its job's home node, so its calls wait while that node is unhealthy.",[426,663,664,665,668],{},"Each server answers ",[339,666,667],{},"GET \u002Fhealthz"," on its edge port (7443) and its management port (8443), for load balancers and monitoring.",[326,670,672],{"id":671},"logs","Logs",[426,674,675],{},"Servers and nodes log to stderr. Raise the level when you need detail:",[331,677,679],{"className":333,"code":678,"filename":335,"language":336,"meta":337,"style":337},"orchestrator-zero server start --log-level debug ...\norchestrator-zero-node run --log-level debug ...\n",[339,680,681,700],{"__ignoreMap":337},[342,682,683,685,688,691,694,697],{"class":344,"line":345},[342,684,274],{"class":348},[342,686,687],{"class":351}," server",[342,689,690],{"class":351}," start",[342,692,693],{"class":351}," --log-level",[342,695,696],{"class":351}," debug",[342,698,699],{"class":351}," ...\n",[342,701,702,704,707,709,711],{"class":344,"line":362},[342,703,278],{"class":348},[342,705,706],{"class":351}," run",[342,708,693],{"class":351},[342,710,696],{"class":351},[342,712,699],{"class":351},[426,714,715,716,719],{},"Temporal's own logs are quieter by default; set ",[339,717,718],{},"--temporal-log-level"," on the server to see them.",[426,721,722,723,726],{},"The edge tells the node its verdict on the control stream, and the node's log says what it does about it (",[339,724,725],{},"health verdict state=degraded ...",").",[728,729,730,734],"note",{},[731,732,733],"strong",{},"Planned (M6):"," logs that stream from nodes to the edge.",[736,737,738],"style",{},"html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":337,"searchDepth":362,"depth":362,"links":740},[741,742,743],{"id":328,"depth":362,"text":329},{"id":431,"depth":362,"text":432},{"id":671,"depth":362,"text":672},"See which nodes are connected, what they run and what state they are in.","md",null,{},{"icon":247},{"title":244,"description":744},"G5N5n1zz7FqAQSIBJ7iSmNuS7R9asKf3_QGZwikBlyY",[752,754],{"title":239,"path":240,"stem":241,"description":753,"icon":242,"children":-1},"Tokens and money per call, job, agent and model, and per branch of a job tree.",{"title":249,"path":250,"stem":251,"description":755,"icon":252,"children":-1},"Every step of every job, as a trace you can read from the CLI and the web UI, with findings about what looks wrong.",1791135413129]