antcolony

55.3 KB

1. // work.hl — `work <project> <n>`: agent v0 (ticket antcolony#2, mission 016, concept
2. // docs/scheduler-agent.md §3 + build order 3) + the controller step (mission 019, build order 4).
3. // Run BY HAND (or by `cycle`) on the host that holds the project's dev folder; ONE worker, no daemon.
4. //
5. // 1. refuse unless: the project takes workers, has a concept, the ticket is `open` (or `rejected` with a
6. // rejection newer than the last colony session — mission 022) and neither blocked nor a parent, the dev folder is
7. // on THIS host and exists, a token is there (COLONY_TOKEN_FILE) — nothing is written before that;
8. // `--dry-run` stops here (prints what would run; no write, no Claude session);
9. // 2. the brief (`brief`, same text) → runs/<session>/brief.md;
10. // 3. the LEASE: ticket → `in progress` with the text "A worker started on this ticket." + the colony-lease line;
11. // 4. Claude Code headless in the dev folder: `claude -p` (brief on stdin) `--output-format json`
12. // `--json-schema` templates/report.schema.json, model / max turns / permission mode configurable
13. // (never bypassPermissions), `--permission-prompts none` (whatever would ask is denied),
14. // `--strict-mcp-config` (no MCP servers), a fixed Claude `--session-id` (recorded before the start);
15. // 5. on exit: raw result → runs/<session>/claude-result.json, the structured output →
16. // runs/<session>/report.json, `report` validation (+ ticket/session must be this session's);
17. // 6. no valid report (no structured output, max turns, timeout, crash, refused by validation) →
18. // ticket back to `open` with the reason (the lease is given back);
19. // 7. a valid report → the CONTROLLER (controller.hl) → with --post: posted by the verdict (pass →
20. // report comment + awaiting creator; fail → findings + open; Nth fail → creator question + on hold).
21. // No verdict (controller crashed / timed out / refused) → nothing posted, ticket stays leased,
22. // `./colony check runs/<session> --post` retries.
23. // Mission 020: the lease carries an expiry (lease.hl) and a heartbeat renews it while the session runs;
24. // a session that ends on the Claude usage limit — or is stopped by `run`'s second SIGTERM — is PARKED
25. // (ticket stays leased, `colony-parked:` marker) and `runResume` continues it later with `claude --resume`
26. // (worker) or a new controller session; `hooks` = { onProc(process, phase), parkReason() } let `run` see
27. // and stop the running process; the port range goes to the session as env COLONY_PORTS / _FROM / _TO.
28. import { env } from 'hl:proc'
29. import { readFile, writeFile, mkDir, exists, remove } from 'hl:fs'
30. import { now, timestamp } from 'hl:time'
31. import { buildBrief, portsOf, PORT_FROM_MARKER, PORT_TO_MARKER } from './brief.hl'
32. import { reportFile, parseRef } from './report.hl'
33. import { readToken, postJson, getJson, ticketPath, pageUrl, baseUrl } from './tickets.hl'
34. import { sandboxFor, deployMarked } from './sandbox.hl'
35. import { creatorNames } from './librarian.hl'
36. import { runClaude, readClaude, usageLine, usageMeta, setting, isWholeNumber, uuid, round4, MODES, rateLimitOf, lastRateInfo, portEnv } from './claude.hl'
37. import { leaseSettings, leaseLine, parkLine, startHeartbeat, postLease, clock } from './lease.hl'
38. import { runController, postByVerdict, loadVerdict, maxFailsOf } from './controller.hl'
39. import { NL, fromCwd, positionals, readJson, option } from './util.hl'
40. import { loadRegistry } from './registry.hl'
41. import { blockReason, questionWhy } from './relations.hl'
43. static hostName = () => {
44. let h = env('COLONY_HOST')
45. if (h == null || h == '') { h = exists('/etc/hostname') ? readFile('/etc/hostname').trim() : '' }
46. return h
47. }
49. static runsDir = (home) => {
50. let runsText = env('COLONY_RUNS_DIR')
51. return runsText == null || runsText == '' ? home + '/runs' : fromCwd(runsText)
52. }
54. // no daemon around: nothing watches the process, nothing asks to park
55. static noHooks = () => {
56. return { onProc = (p, phase) => { return null } parkReason = () => { return null } }
57. }
59. // costs of a session over all its parts (a parked + resumed session has several)
60. static workerCostOf = (meta) => {
61. let a = meta.costEarlier == null ? 0 : meta.costEarlier
62. if (meta.costUsd == null) { return meta.costEarlier }
63. return a + meta.costUsd
64. }
65. static controllerCostOf = (meta) => {
66. let a = meta.controllerCostEarlier
67. let c = meta.controller == null ? null : meta.controller.costUsd
68. if (c == null) { return a }
69. if (a == null) { return c }
70. return a + c
71. }
73. // antcolony#18: the box a session of this run lives in (sandbox.hl). No box possible → a command that fails at once
74. // (never a session with host access because the box could not be built).
75. static boxOf = (meta, mayDeploy, more) => {
76. let r = sandboxFor({ project = meta.project folder = meta.folder host = meta.host deploy = mayDeploy extraRo = more })
77. if (r.error != null) {
78. console.log('work: SANDBOX — ' + r.error + ' — the session cannot start')
79. return ['/usr/bin/false' '--']
80. }
81. return r.args
82. }
84. // `box <project> [--deploy] [--folder F]` (antcolony#18): the argv prefix of the sandbox a session of that project runs in,
85. // one argument per line (a test / the architect runs `<prefix> cmd …` to see what a worker sees). --folder = a dev folder
86. // other than the registry's.
87. static runBox = (argv) => {
88. let pos = positionals(argv)
89. if (pos.length != 2) {
90. console.log('usage: box <project> [--deploy] [--folder F]')
91. return false
92. }
93. let reg = loadRegistry()
94. let m = reg.projects[pos[1]]
95. let folder = option(argv, '--folder')
96. if (folder == null || folder == '') { folder = m == null || m.dev == null ? null : m.dev.folder }
97. if (folder == null) {
98. console.log('box: REFUSED — project ' + pos[1] + ' has no dev folder')
99. return false
100. }
101. let r = sandboxFor({ project = pos[1] folder = folder host = hostName() deploy = argv.includes('--deploy') extraRo = null })
102. if (r.error != null) {
103. console.log('box: REFUSED — ' + r.error)
104. return false
105. }
106. for (x of r.args) { console.log(x) }
107. return true
108. }
110. // PARK a session (mission 020): the ticket stays leased (`in progress`), a comment with the reason and the
111. // `colony-parked:` marker (resume time, lease expiry = resume + lease ttl); `run` on this host resumes it
112. // after `resumeMs` once the quota is below the reserve. phase = worker (→ claude --resume) | controller
113. // (→ a new controller session on the kept report). Calls done({ outcome: 'parked', … }).
114. static parkRun = (meta, phase, reason, resumeMs, resetKnown, done) => {
115. let prev = meta.parked == null ? 0 : meta.parked.count
116. if (phase == 'worker') {
117. meta.costEarlier = workerCostOf(meta)
118. meta.costUsd = null
119. } else if (meta.controller != null && meta.controller.costUsd != null) {
120. meta.controllerCostEarlier = controllerCostOf(meta)
121. meta.controller.costUsd = null
122. }
123. let count = prev + 1
124. meta.parked = { phase = phase reason = reason resumeMs = resumeMs resumeAfter = timestamp(resumeMs) count = count at = timestamp(now()) }
125. meta.outcome = 'parked (' + phase + ')'
126. writeFile(meta.runDir + '/session.json', JSON.stringify(meta), 384)
127. let ttl = meta.leaseTtl == null ? 3600 : meta.leaseTtl
128. let exp = resumeMs + ttl * 1000
129. // mission 023: one plain line for the creator + the machine line (the Claude session id is in session.json)
130. let why = reason.startsWith('Claude usage limit') ? 'the Claude usage limit is reached' : 'the scheduler was stopped'
131. let text = 'Paused: ' + why + ' — work continues after ' + clock(resumeMs) + '.' + NL + NL + parkLine(meta.session, meta.host, phase, resumeMs, exp)
132. let ref = parseRef(meta.ticket)
133. let tk = readToken()
134. let posted = 'no token: ' + tk.error
135. if (tk.error == null) {
136. let r = postLease(ref.project, ref.number, text, tk.token)
137. posted = r.ok ? 'marker posted' : 'POSTING THE MARKER FAILED: ' + r.text
138. }
139. console.log('PARKED — ' + meta.ticket + ' session ' + meta.session + ' (' + phase + ') until ' + timestamp(resumeMs) + ': ' + reason + ' · ' + posted + ' · files ' + meta.runDir + '/')
140. // mission 034: `run` pauses the librarian + starts until the reset when the park was the usage limit
141. let lim = reason.startsWith('Claude usage limit')
142. done({ outcome = 'parked' session = meta.session workerCost = workerCostOf(meta) controllerCost = controllerCostOf(meta) limited = lim resetMs = lim && resetKnown ? resumeMs : null limitText = lim ? reason : null })
143. return null
144. }
146. // the controller's settings (options / env / defaults; the worker's permission mode and claude binary)
147. static controllerSettings = (argv) => {
148. let claudeBin = setting(argv, '--claude', 'COLONY_CLAUDE', 'claude')
149. return {
150. bin = setting(argv, '--controller-claude', 'COLONY_CONTROLLER_CLAUDE', claudeBin)
151. model = setting(argv, '--controller-model', 'COLONY_CONTROLLER_MODEL', 'sonnet')
152. maxTurns = setting(argv, '--controller-max-turns', 'COLONY_CONTROLLER_MAX_TURNS', '20')
153. timeout = setting(argv, '--controller-timeout', 'COLONY_CONTROLLER_TIMEOUT', '900')
154. mode = setting(argv, '--permission-mode', 'COLONY_PERMISSION_MODE', 'auto')
155. budget = setting(argv, '--max-budget-usd', 'COLONY_MAX_BUDGET_USD', '')
156. maxFails = maxFailsOf(setting(argv, '--max-fails', 'COLONY_MAX_FAILS', '2'))
157. }
158. }
160. // refusals shared by work and check → text or null
161. static controllerRefusal = (cs) => {
162. if (!MODES.includes(cs.mode)) { return 'permission mode "' + cs.mode + '" is not allowed (allowed: ' + MODES.join(', ') + '; never bypassPermissions)' }
163. if (!isWholeNumber(cs.maxTurns) || cs.maxTurns == '0') { return '--controller-max-turns must be a whole number ≥ 1, got ' + cs.maxTurns }
164. if (!isWholeNumber(cs.timeout) || cs.timeout == '0') { return '--controller-timeout must be whole seconds ≥ 1, got ' + cs.timeout }
165. return null
166. }
168. // 7. the controller's answer → post by it (or not) → the last line; calls done(result)
169. // x = { ref, dir, post, meta, report, cs, label ('WORK' / 'CHECK') }
170. static afterVerdict = (x, cv, done) => {
171. let meta = x.meta
172. let pr = x.hooks == null ? null : x.hooks.parkReason()
173. if (pr != null) {
174. parkRun(meta, 'controller', pr, now(), false, done)
175. return null
176. }
177. if (cv.limited == true) {
178. parkRun(meta, 'controller', 'Claude usage limit hit by the controller (' + cv.limitText + ')', cv.resetMs == null ? now() + 3600000 : cv.resetMs, cv.resetMs != null, done)
179. return null
180. }
181. let res = { outcome = null session = meta.session workerCost = workerCostOf(meta) controllerCost = controllerCostOf(meta) }
182. if (cv.verdict == null) {
183. meta.outcome = 'valid report, no verdict: ' + cv.reason
184. writeFile(x.dir + '/session.json', JSON.stringify(meta), 384)
185. console.log(x.label + ' OK, CONTROLLER FAILED — ' + cv.reason + '; nothing posted, ' + x.ref + ' stays "in progress" (leased). Retry: ./colony check ' + x.dir + (x.post ? ' --post' : ''))
186. res.outcome = 'no verdict'
187. done(res)
188. return null
189. }
190. if (!x.post) {
191. meta.outcome = 'valid report, verdict ' + cv.effective + ', not posted'
192. writeFile(x.dir + '/session.json', JSON.stringify(meta), 384)
193. console.log(x.label + ' OK — valid report, controller: ' + cv.effective + ', NOT posted (no --post); ' + x.ref + ' stays "in progress" (leased). Post it by the verdict: ./colony check ' + x.dir + ' --post')
194. res.outcome = 'verdict ' + cv.effective + ', not posted'
195. done(res)
196. return null
197. }
198. let p = postByVerdict(x.report, cv, meta, x.cs.maxFails)
199. meta.outcome = p.outcome
200. writeFile(x.dir + '/session.json', JSON.stringify(meta), 384)
201. res.outcome = p.outcome
202. if (!p.ok) {
203. console.log(x.label + ' OK, POST FAILED — controller: ' + cv.effective + '; re-run ./colony check ' + x.dir + ' --post (idempotent). ' + x.ref + ' stays "in progress".')
204. } else if (p.outcome == 'posted') {
205. console.log(x.label + ' DONE — ' + x.ref + ' report posted (controller: pass) · files ' + x.dir + '/')
206. } else if (p.outcome == 'ready to deploy') {
207. // mission 027: the deploy gate — the creator tests on the live site, so not "awaiting creator" before the deploy
208. let rr = parseRef(x.ref)
209. console.log(x.label + ' DONE — ' + x.ref + ' built, waiting for the deploy (controller: pass): "Built — goes live with the next deploy.", stays in progress; after the deploy: ./colony deployed ' + rr.project + ' ' + rr.number + ' · files ' + x.dir + '/')
210. } else if (p.outcome == 'half done') {
211. console.log(x.label + ' HALF DONE — ' + x.ref + ': the report says the work is not complete → "Half done" comment, back to open, the next worker continues (controller: pass) · files ' + x.dir + '/')
212. } else if (p.outcome == 'sent back') {
213. console.log(x.label + ' FAILED — controller: fail (' + p.fails + ' of ' + x.cs.maxFails + ') → ' + x.ref + ' back to open with the findings · files ' + x.dir + '/')
214. } else if (p.outcome == 'question') {
215. console.log(x.label + ' FAILED — controller: fail (' + p.fails + ' of ' + x.cs.maxFails + ') → creator question ' + p.question.project + p.question.ref + ', ' + x.ref + ' on hold · files ' + x.dir + '/')
216. } else {
217. console.log(x.label + ' — controller: ' + cv.effective + ', ' + p.outcome + ' · files ' + x.dir + '/')
218. }
219. done(res)
220. return null
221. }
223. // ---- mission 025: the FINALIZE step ------------------------------------------------------------------------------
224. // A worker that ran out of steps (`error_max_turns`) or time (killed by --timeout) did work but wrote no report (first
225. // live cycle, ident.worldapi.org#2: all gates green in its dev copy, then max turns before the copy-back + report →
226. // $4.67 for nothing). Before the lease goes back, the SAME Claude session is resumed ONCE (`claude -p --resume <id>`,
227. // same model / permission mode / --json-schema, a small --max-turns) with a short instruction: stop, start nothing new,
228. // change nothing, write the report about what is done / verified / open.
229. // Mission 026: NO `--disallowedTools` any more — it changes the tool list, i.e. the cached prompt prefix, so the first
230. // real finalize re-sent the whole 207 k context without cache ($1.05). The "change nothing" instruction + auto mode stay.
231. // A valid report → controller → post as usual; none → back to open as before (once). The cost is summed.
233. // --finalize-max-turns (default 8; 0 = no finalize step) / --finalize-timeout (default 600 s, never more than the
234. // worker's own --timeout) → { maxTurns, timeout, error }
235. static finalizeSettings = (argv, workerTimeout) => {
236. let maxTurns = setting(argv, '--finalize-max-turns', 'COLONY_FINALIZE_MAX_TURNS', '8')
237. let wt = isWholeNumber(workerTimeout) ? toNumber(workerTimeout) : 600
238. let deflt = wt < 600 ? '' + wt : '600'
239. let timeout = setting(argv, '--finalize-timeout', 'COLONY_FINALIZE_TIMEOUT', deflt)
240. let error = null
241. if (!isWholeNumber(maxTurns)) { error = '--finalize-max-turns must be a whole number (0 = no finalize step), got ' + maxTurns }
242. else if (!isWholeNumber(timeout) || timeout == '0') { error = '--finalize-timeout must be whole seconds ≥ 1, got ' + timeout }
243. return { maxTurns = maxTurns timeout = timeout error = error }
244. }
246. // why the worker stopped, when a finalize step can help → text or null
247. static finalizeWhy = (res, timedOut) => {
248. if (timedOut) { return 'ran out of time' }
249. if (res != null && res.subtype == 'error_max_turns') { return 'ran out of steps' }
250. return null
251. }
253. // mission 027 ("validate → reject incomplete reports back to the worker (one retry)", concept §2): a report the validation
254. // refused (e.g. a result over 120 characters, a test step on a work copy, 4 questions) gets the SAME Claude session resumed
255. // once — like the finalize step, with the problems — to answer the corrected report; the work is not lost for a long line
256. static REFUSED_WHY = 'had its report refused'
257. static fixPrompt = (meta, maxTurns, problems) => {
258. let o = []
259. o.push('# FIX YOUR REPORT — the scheduler refused it')
260. o.push('')
261. o.push('Your report was refused by the validation (' + problems.length + ' problem' + (problems.length == 1 ? '' : 's') + '):')
262. for (p of problems) { o.push('- ' + p) }
263. o.push('')
264. o.push('- **Do not continue the task.** Start nothing new, change no file (no Edit/Write, nothing through the shell). You have at most ' + maxTurns + ' steps — answer right away.')
265. o.push('- Answer the SAME report again with only these problems fixed: shorten `result` (≤ 120 characters) / the `test` steps (2–4, ≤ 90 characters each, for the creator on the real / live site, never a work copy), ask at most 3 short questions (decide small things yourself and list them in `decided`), one line per `decided` entry. Everything else stays as it was.')
266. o.push('')
267. o.push('Answer with the JSON report exactly as the brief (the first message of this session) says, with')
268. o.push(' "ticket": "' + meta.ticket + '",')
269. o.push(' "session": "' + meta.session + '",')
270. o.push('')
271. o.push('colony-finalize: ' + meta.session)
272. return o.join(NL)
273. }
275. // the finalize instruction (stdin of the resumed session) — kept as runs/<S>/finalize.md
276. static finalizePrompt = (meta, why, maxTurns) => {
277. let o = []
278. o.push('# STOP WORKING — write your report now')
279. o.push('')
280. o.push('Your session ' + why + ' (' + (why == 'ran out of steps' ? 'the step limit of ' + meta.maxTurns + ' was reached' : 'killed after ' + meta.timeoutSeconds + ' s') + ') before you wrote your report. It is resumed ONLY so that you write it — everything you did is lost for the creator unless it is in the report.')
281. o.push('')
282. o.push('- **Do not continue the task.** Start nothing new: no edits, no copying, no deleting, no servers, no tests, no git. Do not use the Edit/Write tools and do not change any file with the shell either.')
283. o.push('- At most a few quick reads (`ls`, `cat`, `grep`) if you must check what exists. You have at most ' + maxTurns + ' steps — better answer right away.')
284. o.push('- Report the truth about THIS session: `done` = what you really did and WHERE (e.g. only in a dev copy, not copied back); `verified` = only checks you really ran earlier in this session, with the command and the output you saw then; everything unfinished goes into `open` (e.g. "the changes are still only in the dev copy", "README/STATUS not updated"). `result` = one plain sentence saying what is done and what is not.')
285. o.push('- `complete`: `true` only if the goal of the ticket is reached AND what the creator should test is really in place (copied into the app, not only a work copy); else `false` — then say in `open` what is missing and WHERE the unfinished work is (e.g. the folder of the work copy), so the next worker can continue.')
286. o.push('- Anything you started that is still running goes into `running`.')
287. o.push('')
288. o.push('Answer with the JSON report exactly as the brief (the first message of this session) says, with')
289. o.push(' "ticket": "' + meta.ticket + '",')
290. o.push(' "session": "' + meta.session + '",')
291. o.push('')
292. o.push('colony-finalize: ' + meta.session)
293. return o.join(NL)
294. }
296. // start the finalize step for a worker that ended without a report; `finish` runs again on its exit (w.finalized)
297. // (`meta` = the caller's session object: Hybriel copies objects stored in fields, so w.meta may be stale — hybriel#38)
298. static startFinalize = (w, meta, why, reason, problems) => {
299. let dir = w.dir
300. meta.finalize = { why = why workerReason = reason workerSubtype = meta.subtype workerTurns = meta.numTurns workerCostUsd = meta.costUsd maxTurns = w.fs.maxTurns timeoutSeconds = w.fs.timeout started = timestamp(now()) outcome = 'running' }
301. if (meta.claudeCostSeen == null) { meta.claudeCostSeen = meta.costUsd }
302. meta.costEarlier = workerCostOf(meta)
303. meta.costUsd = null
304. meta.outcome = 'finalizing (the worker ' + why + ')'
305. // the worker's own output stays beside the finalize output (the controller sees both parts)
306. if (exists(w.outFile)) {
307. writeFile(dir + '/claude-result.before-finalize.json', readFile(w.outFile), 384)
308. remove(w.outFile)
309. }
310. if (exists(w.errFile)) {
311. writeFile(dir + '/claude-stderr.before-finalize.log', readFile(w.errFile), 384)
312. remove(w.errFile)
313. }
314. if (exists(w.reportPath)) { remove(w.reportPath) }
315. writeFile(dir + '/session.json', JSON.stringify(meta), 384)
316. let promptFile = dir + '/finalize.md'
317. writeFile(promptFile, why == REFUSED_WHY ? fixPrompt(meta, w.fs.maxTurns, problems) : finalizePrompt(meta, why, w.fs.maxTurns), 384)
318. let args = ['-p' '--resume' meta.claudeSession '--output-format' 'json' '--json-schema' readFile('./templates/report.schema.json') '--model' meta.model '--max-turns' w.fs.maxTurns '--permission-mode' meta.permissionMode '--permission-prompts' 'none' '--strict-mcp-config']
319. if (meta.maxBudgetUsd != null && meta.maxBudgetUsd != '') {
320. args.push('--max-budget-usd')
321. args.push(meta.maxBudgetUsd)
322. }
323. w.finalized = true
324. w.resumedPart = true
325. w.firstReason = reason
326. w.firstProblems = problems == null ? [] : problems
327. w.firstTimedOut = why == 'ran out of time'
328. w.timeoutText = w.fs.timeout
329. w.meta = meta
330. console.log('work: FINALIZE — the worker ' + (why == REFUSED_WHY ? why + ' (' + reason + '); resuming claude session ' + meta.claudeSession + ' once to FIX it' : why + ' without a report (' + reason + '); resuming claude session ' + meta.claudeSession + ' once to write it') + ': --max-turns ' + w.fs.maxTurns + ', timeout ' + w.fs.timeout + ' s, same tools as the worker (prompt cache) · ' + promptFile)
331. let boxed = boxOf(meta, meta.deploy == true, null)
332. let p = runClaude({ label = 'finalize' folder = meta.folder inFile = promptFile outFile = w.outFile errFile = w.errFile bin = meta.claude args = args timeout = w.fs.timeout env = portEnv(meta.ports) session = meta.session ports = meta.ports sandbox = boxed }, (x) => {
333. finish(w, x)
334. return null
335. })
336. w.hooks.onProc(p, 'worker')
337. return p
338. }
340. // 5./6. — after the worker's exit (or, mission 025, after the finalize step's exit: w.finalized)
341. static finish = (w, x) => {
342. let meta = w.meta
343. let fin = w.finalized == true
344. meta.ended = timestamp(now())
345. meta.exit = x.ev.exit
346. for (l of x.lines) { console.log((fin ? ' finalize ' : ' worker ') + l) }
347. // mission 026: what this part left listening on its ports was stopped (claude.hl cleanupPorts) — kept in session.json
348. if (x.cleanup.length > 0) {
349. let cl = []
350. if (meta.cleanup != null) { for (c of meta.cleanup) { cl.push(c) } }
351. cl.push({ part = fin ? 'finalize' : 'worker' at = timestamp(now()) lines = x.cleanup })
352. meta.cleanup = cl
353. }
354. let rc = readClaude(w.outFile, x.ev.exit, x.timedOut, w.timeoutText == null ? meta.timeoutSeconds : w.timeoutText, fin ? 'finalize step' : 'worker')
355. let reason = rc.reason
356. if (rc.cr != null) {
357. meta.permissionModeSeen = rc.cr.mode
358. meta.permissionModes = rc.cr.modes
359. if (rc.cr.mode != null && rc.cr.mode != w.mode) { console.log('work: WARNING — asked for permission mode "' + w.mode + '", Claude reported "' + rc.cr.mode + '" (' + rc.cr.modes.join(', ') + ') — e.g. auto is not available for every model') }
360. }
361. let res = rc.res
362. if (res != null) {
363. let um = usageMeta(res)
364. for (k of um.keys()) { meta[k] = um[k] }
365. console.log(usageLine(fin ? 'FINALIZE USAGE' : 'USAGE', res))
366. // mission 025 (seen on the first real finalize, Claude Code 2.1.280): a RESUMED session reports the WHOLE session's
367. // cost in total_cost_usd / modelUsage (cumulative over all its parts), not this part's → this part = the total now
368. // minus the total seen after the previous part (meta.claudeCostSeen); a smaller total → taken as this part's own
369. let total = res.total_cost_usd
370. if (w.resumedPart == true && total != null && meta.claudeCostSeen != null) {
371. let inc = round4(total - meta.claudeCostSeen)
372. if (inc < 0) { inc = total }
373. meta.costUsd = inc
374. meta.claudeCostTotal = total
375. console.log((fin ? 'FINALIZE' : 'RESUME') + ' COST: $' + inc + ' for this part (Claude reports $' + total + ' for the whole session so far)')
376. }
377. if (total != null) { meta.claudeCostSeen = total }
378. if (res.permission_denials != null && res.permission_denials.length > 0) { console.log('work: ' + res.permission_denials.length + ' permission denial(s) — see ' + w.dir + '/session.json') }
379. if (res.structured_output == null) {
380. if (reason == null) { reason = 'no structured report (Claude: ' + res.subtype + (res.is_error ? ', error' : '') + ', ' + res.num_turns + ' turns, exit ' + x.ev.exit + ')' }
381. } else {
382. writeFile(w.reportPath, JSON.stringify(res.structured_output), 384)
383. }
384. }
385. // mission 020: stopped by `run` (second SIGTERM) or evicted by the usage limit → PARK, not give back
386. let ri = lastRateInfo(rc.value)
387. if (ri != null) { meta.rateLimit = ri }
388. let pr = w.hooks.parkReason()
389. let rl = rateLimitOf(rc.value, res)
390. if (reason != null && pr != null) {
391. parkRun(meta, 'worker', pr, now(), false, w.done)
392. return false
393. }
394. if (reason != null && rl.limited) {
395. parkRun(meta, 'worker', 'Claude usage limit (' + rl.text + ')', rl.resetMs == null ? now() + 3600000 : rl.resetMs, rl.resetMs != null, w.done)
396. return false
397. }
398. // mission 025: out of steps / time → resume the same session ONCE to write the report
399. if (reason != null && !fin && w.fs != null && w.fs.maxTurns != '0' && meta.claudeSession != null) {
400. let fw = finalizeWhy(res, x.timedOut)
401. if (fw != null) {
402. startFinalize(w, meta, fw, reason, null)
403. return true
404. }
405. }
406. if (fin) {
407. meta.finalize.ended = timestamp(now())
408. meta.finalize.costUsd = meta.costUsd
409. meta.finalize.numTurns = res == null ? null : res.num_turns
410. meta.finalize.subtype = res == null ? null : res.subtype
411. meta.finalize.outcome = reason == null ? 'report written' : 'no report: ' + reason
412. if (reason != null) { console.log('work: FINALIZE FAILED — ' + reason + ' (the worker ' + meta.finalize.why + ' before)') }
413. }
414. let problems = []
415. if (reason == null) {
416. let rep = reportFile(w.reportPath, false, { ticket = w.ref session = w.session })
417. if (fin) { meta.finalize.outcome = rep.valid ? 'valid report' : 'report refused by the validation' }
418. if (!rep.valid) {
419. reason = 'the report was refused by the validation (' + rep.problems.length + ' problem' + (rep.problems.length == 1 ? '' : 's') + ')'
420. problems = rep.problems
421. // mission 027: ONE retry — the same session fixes its report (not after a finalize / fix step: one per session)
422. if (!fin && pr == null && w.fs != null && w.fs.maxTurns != '0' && meta.claudeSession != null) {
423. startFinalize(w, meta, REFUSED_WHY, reason, problems)
424. return true
425. }
426. } else if (pr != null) {
427. parkRun(meta, 'controller', pr + ' (the worker had finished with a valid report; the controller runs on resume)', now(), false, w.done)
428. return false
429. } else {
430. // 7. the controller
431. meta.outcome = 'valid report, controller running'
432. writeFile(w.dir + '/session.json', JSON.stringify(meta), 384)
433. let report = readJson(w.reportPath).value
434. let cs = w.cs
435. let boxed = boxOf(meta, false, [w.dir])
436. let cp = runController({ dir = w.dir folder = meta.folder ref = w.ref session = w.session host = w.host ports = meta.ports bin = cs.bin model = cs.model maxTurns = cs.maxTurns mode = cs.mode timeout = cs.timeout budget = cs.budget note = w.controllerNote sandbox = boxed }, meta, (cv) => {
437. afterVerdict({ ref = w.ref dir = w.dir post = w.post meta = meta report = report cs = cs label = w.label == null ? 'WORK' : w.label hooks = w.hooks }, cv, w.done)
438. return null
439. })
440. w.hooks.onProc(cp, 'controller')
441. return true
442. }
443. }
444. // no valid report → the lease goes back: ticket → open, with the reason
445. // (mission 025: after a failed finalize step the WORKER's reason counts; the finalize step's is added)
446. let finReason = null
447. let timedOut = x.timedOut
448. if (fin) {
449. finReason = reason
450. reason = w.firstReason
451. timedOut = w.firstTimedOut
452. if (problems.length == 0 && w.firstProblems != null) { problems = w.firstProblems }
453. }
454. meta.outcome = 'no valid report: ' + reason + (fin ? ' · finalize step: ' + finReason : '')
455. meta.problems = problems
456. writeFile(w.dir + '/session.json', JSON.stringify(meta), 384)
457. // mission 023: plain words for the creator; the reason in full + the validation problems stay in session.json
458. let why = 'it ended without a result'
459. if (timedOut) { why = 'it ran out of time' }
460. else if (reason.includes('error_max_turns')) { why = 'it ran out of steps' }
461. else if (problems.length > 0) { why = 'its report was incomplete' }
462. let text = 'Back to open: the worker stopped without a usable result (' + why + ') — the next worker tries again.' + NL + NL + 'colony-run: ' + w.session + ' · host ' + w.host + ' · no report'
463. let s = postJson(ticketPath(w.project, w.number) + '/state', { state = 'open' text = text }, w.token)
464. // `salvage` (mission 025; since mission 026 it holds a lease like work → gives it back the same way)
465. let head = w.salvage == true ? 'SALVAGE FAILED — the finalize step gave no valid report (' + finReason + '); the worker: ' + reason : 'WORK FAILED — ' + reason
466. if (s.status != 201) {
467. console.log(head + '; AND giving the lease back failed: ' + s.status + ' ' + s.text + ' — set ' + w.ref + ' to open by hand')
468. } else {
469. console.log(head + ' → ' + w.ref + ' back to open · files ' + w.dir + '/')
470. }
471. w.done({ outcome = 'no valid report' session = w.session workerCost = workerCostOf(meta) controllerCost = null })
472. return false
473. }
475. static noop = (r) => { return null }
477. // `work`; `done(result)` is called once the whole run is over (result = { outcome, workerCost,
478. // controllerCost }) — not on a refusal / dry run. → true when a worker was started
479. static runWork = (argv, whenDone, hooksIn) => {
480. let done = whenDone == null ? noop : whenDone
481. let hooks = hooksIn == null ? noHooks() : hooksIn
482. let pos = positionals(argv)
483. if (pos.length != 3) {
484. console.log('usage: work <project> <number> [--post] [--dry-run] [--model M] [--max-turns N] [--permission-mode MODE] [--timeout SECONDS] [--max-budget-usd X] [--ports A-B] [--session ID] [--controller-model M] [--controller-max-turns N] [--controller-timeout S] [--max-fails N]')
485. return false
486. }
487. let project = pos[1]
488. let number = pos[2]
489. let post = argv.includes('--post')
490. let dry = argv.includes('--dry-run')
491. let model = setting(argv, '--model', 'COLONY_MODEL', 'sonnet')
492. let maxTurns = setting(argv, '--max-turns', 'COLONY_MAX_TURNS', '40')
493. let mode = setting(argv, '--permission-mode', 'COLONY_PERMISSION_MODE', 'auto')
494. let timeoutText = setting(argv, '--timeout', 'COLONY_WORK_TIMEOUT', '3600')
495. let budget = setting(argv, '--max-budget-usd', 'COLONY_MAX_BUDGET_USD', '')
496. let claudeBin = setting(argv, '--claude', 'COLONY_CLAUDE', 'claude')
497. let cs = controllerSettings(argv)
498. let home = env('COLONY_HOME')
499. if (home == null || home == '') {
500. console.log('work: REFUSED — COLONY_HOME is not set (run through ./colony, which sets it)')
501. return false
502. }
503. if (!MODES.includes(mode)) {
504. console.log('work: REFUSED — permission mode "' + mode + '" is not allowed (allowed: ' + MODES.join(', ') + '; never bypassPermissions)')
505. return false
506. }
507. if (!isWholeNumber(maxTurns) || maxTurns == '0') {
508. console.log('work: REFUSED — --max-turns must be a whole number ≥ 1, got ' + maxTurns)
509. return false
510. }
511. if (!isWholeNumber(timeoutText) || timeoutText == '0') {
512. console.log('work: REFUSED — --timeout must be whole seconds ≥ 1, got ' + timeoutText)
513. return false
514. }
515. let cr = controllerRefusal(cs)
516. if (cr != null) {
517. console.log('work: REFUSED — ' + cr)
518. return false
519. }
520. let ls = leaseSettings(argv)
521. if (ls.error != null) {
522. console.log('work: REFUSED — ' + ls.error)
523. return false
524. }
525. let fs = finalizeSettings(argv, timeoutText)
526. if (fs.error != null) {
527. console.log('work: REFUSED — ' + fs.error)
528. return false
529. }
530. let tk = readToken()
531. if (tk.error != null) {
532. console.log('work: REFUSED — ' + tk.error + ' (work writes the lease, so it always needs the token)')
533. return false
534. }
535. let b = buildBrief(project, number, argv)
536. if (b.error != null) {
537. console.log('work: ' + b.error)
538. return false
539. }
540. // mission 035 (antcolony#14): the scheduler assembles the brief (it holds the creator's decisions); an agent's `work`
541. // takes that text and fills in ITS port range. The ticket checks below still run here, on this host.
542. let briefFrom = option(argv, '--brief-file')
543. if (briefFrom != null && briefFrom != '') {
544. if (!exists(briefFrom)) {
545. console.log('work: REFUSED — --brief-file ' + briefFrom + ' does not exist')
546. return false
547. }
548. let pr = portsOf(argv)
549. b.text = readFile(briefFrom).replaceAll(PORT_FROM_MARKER, '' + pr.from).replaceAll(PORT_TO_MARKER, '' + pr.to)
550. b.ports = pr
551. }
552. let m = b.meta
553. if (!m.takesWorkers) {
554. console.log('work: REFUSED — project ' + project + ' takes no workers ("workers": false in the registry)')
555. return false
556. }
557. if (b.ticket.state == 'rejected') {
558. // mission 022 "Rejections are work": only while the last rejection is newer than the last colony session
559. if (!b.rejection.eligible) {
560. console.log('work: REFUSED — ' + b.ref + ' is "rejected", and no rejection is newer than the last colony session on it')
561. return false
562. }
563. console.log('work: ' + b.ref + ' was REJECTED by the creator (' + b.rejection.rejection.when + ') — reworking it; the brief starts with the rejection')
564. } else if (b.ticket.state != 'open') {
565. console.log('work: REFUSED — ' + b.ref + ' is "' + b.ticket.state + '", not open (in progress = leased by another session)')
566. return false
567. }
568. let br = blockReason(b.rel, b.rejection != null && b.ticket.state == 'rejected')
569. if (br != null) {
570. console.log('work: REFUSED — ' + b.ref + ' is not eligible: ' + br)
571. return false
572. }
573. let host = hostName()
574. if (m.dev == null || m.dev.folder == null || m.dev.folder == '') {
575. console.log('work: REFUSED — project ' + project + ' has no dev folder in the registry')
576. return false
577. }
578. if (m.dev.host != null && m.dev.host.toLowerCase() != host.toLowerCase()) {
579. console.log('work: REFUSED — the dev folder of ' + project + ' is on ' + m.dev.host + ', this host is ' + host + ' (COLONY_HOST overrides the name)')
580. return false
581. }
582. if (!exists(m.dev.folder)) {
583. console.log('work: REFUSED — dev folder ' + m.dev.folder + ' does not exist on ' + host)
584. return false
585. }
586. let session = b.session
587. let dir = runsDir(home) + '/' + session
588. if (exists(dir)) {
589. console.log('work: REFUSED — ' + dir + ' exists already (one session, one run)')
590. return false
591. }
592. if (dry) {
593. console.log('work: DRY RUN — would lease ' + b.ref + ' ("session ' + session + ' started on ' + host + '"), write ' + dir + '/, run the worker ' + claudeBin + ' -p --model ' + model + ' --max-turns ' + maxTurns + ' --permission-mode ' + mode + ' in ' + m.dev.folder + ' (brief ' + b.text.length + ' characters, ports ' + b.ports.from + '-' + b.ports.to + '), ' + (fs.maxTurns == '0' ? 'no finalize step' : 'out of steps / time → finalize step (--max-turns ' + fs.maxTurns + ', ' + fs.timeout + ' s)') + ', then the controller ' + cs.bin + ' -p --model ' + cs.model + ' --max-turns ' + cs.maxTurns + ', then ' + (post ? 'post by the verdict (max fails ' + cs.maxFails + ')' : 'NOT post (no --post)') + '. Nothing written, no Claude session started.')
594. return false
595. }
596. let box = sandboxFor({ project = project folder = m.dev.folder host = host deploy = false extraRo = null })
597. if (box.error != null) {
598. console.log('work: REFUSED — ' + box.error)
599. return false
600. }
601. mkDir(dir, 448)
602. let briefFile = dir + '/brief.md'
603. let outFile = dir + '/claude-result.json'
604. let errFile = dir + '/claude-stderr.log'
605. let reportPath = dir + '/report.json'
606. writeFile(briefFile, b.text, 384)
607. let claudeSession = uuid()
608. let marked = deployMarked(project, b.ticket, b.events, creatorNames())
609. let meta = { ticket = b.ref project = project number = number post = post session = session host = host folder = m.dev.folder runDir = dir ports = b.ports.from + '-' + b.ports.to model = model maxTurns = maxTurns permissionMode = mode timeoutSeconds = timeoutText maxBudgetUsd = budget claude = claudeBin claudeSession = claudeSession leaseTtl = ls.ttl heartbeatSeconds = ls.heartbeat started = timestamp(now()) outcome = 'running' deploy = marked }
610. writeFile(dir + '/session.json', JSON.stringify(meta), 384)
612. // 3. the lease (mission 020: with an expiry; the heartbeat renews it while the session runs)
613. let expiry = now() + ls.ttl * 1000
614. let lease = postJson(ticketPath(project, number) + '/state', { state = 'in progress' text = 'A worker started on this ticket.' + NL + NL + leaseLine(session, host, 'started', expiry) }, tk.token)
615. if (lease.status != 201) {
616. meta.outcome = 'lease failed: ' + lease.status
617. writeFile(dir + '/session.json', JSON.stringify(meta), 384)
618. console.log('work: FAILED — the lease (' + b.ref + ' → in progress) was refused: ' + lease.status + ' ' + lease.text + ' — no worker started')
619. return false
620. }
621. console.log('work: ' + b.ref + ' leased — "session ' + session + ' started on ' + host + '" until ' + timestamp(expiry) + ' (heartbeat every ' + ls.heartbeat + ' s) ' + pageUrl(b.ticket))
622. let hb = startHeartbeat({ project = project number = number ref = b.ref session = session host = host ttl = ls.ttl heartbeat = ls.heartbeat })
623. let userDone = done
624. done = (r) => {
625. hb.stop()
626. userDone(r)
627. return null
628. }
630. // 4. the worker
631. let args = ['-p' '--output-format' 'json' '--json-schema' readFile('./templates/report.schema.json') '--model' model '--max-turns' maxTurns '--permission-mode' mode '--permission-prompts' 'none' '--strict-mcp-config' '--session-id' claudeSession]
632. if (budget != '') {
633. args.push('--max-budget-usd')
634. args.push(budget)
635. }
636. let w = { meta = meta dir = dir outFile = outFile errFile = errFile reportPath = reportPath post = post ref = b.ref session = session host = host project = project number = number token = tk.token ticket = b.ticket mode = mode cs = cs done = done hooks = hooks fs = fs }
637. let boxed = boxOf(meta, meta.deploy == true, null)
638. let p = runClaude({ label = 'work' folder = m.dev.folder inFile = briefFile outFile = outFile errFile = errFile bin = claudeBin args = args timeout = timeoutText env = portEnv(meta.ports) session = session ports = meta.ports sandbox = boxed }, (x) => {
639. finish(w, x)
640. return null
641. })
642. hooks.onProc(p, 'worker')
643. console.log('work: worker started — ' + claudeBin + ' -p --model ' + model + ' --max-turns ' + maxTurns + ' --permission-mode ' + mode + ' in ' + m.dev.folder + ' · pid ' + p.pid + ' · claude session ' + claudeSession + ' · ports ' + meta.ports + ' · files ' + dir + '/')
644. return true
645. }
647. // RESUME a parked session (mission 020; called by `run` once the park time is over and the quota is below the
648. // reserve): lease comment `resumed` + heartbeat, then phase worker → `claude -p --resume <claude session>` with a
649. // short prompt (the brief is in the session already; the port range may be new) → the normal finish (report →
650. // controller → post); phase controller → a new controller session on the kept report.json → post.
651. // `ports` = the range for this part ('A-B', null = keep). → true when a session was started.
652. static runResume = (dir, argv, whenDone, hooksIn, ports) => {
653. let done = whenDone == null ? noop : whenDone
654. let hooks = hooksIn == null ? noHooks() : hooksIn
655. let sj = readJson(dir + '/session.json')
656. if (sj.error != null) {
657. console.log('resume: REFUSED — ' + sj.error)
658. return false
659. }
660. let meta = sj.value
661. if (meta.parked == null) {
662. console.log('resume: REFUSED — ' + dir + ' is not parked (outcome: ' + meta.outcome + ')')
663. return false
664. }
665. let host = hostName()
666. if (meta.host != host) {
667. console.log('resume: REFUSED — session ' + meta.session + ' ran on ' + meta.host + ', this host is ' + host + ' (its Claude session lives there)')
668. return false
669. }
670. let tk = readToken()
671. if (tk.error != null) {
672. console.log('resume: REFUSED — ' + tk.error)
673. return false
674. }
675. let cs = controllerSettings(argv)
676. let k = meta.parked.count
677. let phase = meta.parked.phase
678. let reason = meta.parked.reason
679. if (ports != null) { meta.ports = ports }
680. let ttl = meta.leaseTtl == null ? 3600 : meta.leaseTtl
681. let hbs = meta.heartbeatSeconds == null ? 1200 : meta.heartbeatSeconds
682. let tref = parseRef(meta.ticket)
683. let expiry = now() + ttl * 1000
684. let lr = postLease(tref.project, tref.number, 'Work continues after a pause.' + NL + NL + leaseLine(meta.session, host, 'resumed (park ' + k + ')', expiry), tk.token)
685. if (!lr.ok) {
686. console.log('resume: FAILED — the lease comment on ' + meta.ticket + ' was refused: ' + lr.text + ' — nothing started')
687. return false
688. }
689. meta.resumed = k
690. meta.outcome = 'resumed (' + phase + ')'
691. writeFile(dir + '/session.json', JSON.stringify(meta), 384)
692. let hb = startHeartbeat({ project = tref.project number = tref.number ref = meta.ticket session = meta.session host = host ttl = ttl heartbeat = hbs })
693. let done2 = (r) => {
694. hb.stop()
695. done(r)
696. return null
697. }
698. if (phase == 'controller') {
699. let report = readJson(dir + '/report.json').value
700. let boxed = boxOf(meta, false, [dir])
701. let cp = runController({ dir = dir folder = meta.folder ref = meta.ticket session = meta.session host = host ports = meta.ports bin = cs.bin model = cs.model maxTurns = cs.maxTurns mode = cs.mode timeout = cs.timeout budget = cs.budget sandbox = boxed }, meta, (cv) => {
702. afterVerdict({ ref = meta.ticket dir = dir post = meta.post meta = meta report = report cs = cs label = 'WORK' hooks = hooks }, cv, done2)
703. return null
704. })
705. hooks.onProc(cp, 'controller')
706. console.log('work: RESUMED ' + meta.ticket + ' session ' + meta.session + ' — a new controller session (park ' + k + ') · ports ' + meta.ports)
707. return true
708. }
709. // the worker: keep the output of the part before the park, continue the same Claude session
710. let outFile = dir + '/claude-result.json'
711. let errFile = dir + '/claude-stderr.log'
712. if (exists(outFile)) {
713. writeFile(dir + '/claude-result.parked-' + k + '.json', readFile(outFile), 384)
714. remove(outFile)
715. }
716. if (exists(errFile)) {
717. writeFile(dir + '/claude-stderr.parked-' + k + '.log', readFile(errFile), 384)
718. remove(errFile)
719. }
720. if (exists(dir + '/report.json')) { remove(dir + '/report.json') }
721. let promptFile = dir + '/resume-' + k + '.md'
722. let pt = 'Your session was interrupted: ' + reason + '. It is resumed now (' + timestamp(now()) + ').' + NL + NL + 'Continue the task of your brief (the first message of this session) from where you stopped — do not start over; check what is already done.' + NL + NL + '**Ports for everything you start: ' + meta.ports + ' only** (also in the env as `COLONY_PORTS`; they may differ from the range in the brief).' + NL + NL + 'When you are done, answer with the JSON report exactly as the brief says, with' + NL + ' "ticket": "' + meta.ticket + '",' + NL + ' "session": "' + meta.session + '",' + NL
723. writeFile(promptFile, pt, 384)
724. let args = ['-p' '--resume' meta.claudeSession '--output-format' 'json' '--json-schema' readFile('./templates/report.schema.json') '--model' meta.model '--max-turns' meta.maxTurns '--permission-mode' meta.permissionMode '--permission-prompts' 'none' '--strict-mcp-config']
725. if (meta.maxBudgetUsd != null && meta.maxBudgetUsd != '') {
726. args.push('--max-budget-usd')
727. args.push(meta.maxBudgetUsd)
728. }
729. let fs = finalizeSettings(argv, meta.timeoutSeconds)
730. let w = { meta = meta dir = dir outFile = outFile errFile = errFile reportPath = dir + '/report.json' post = meta.post ref = meta.ticket session = meta.session host = host project = tref.project number = tref.number token = tk.token ticket = null mode = meta.permissionMode cs = cs done = done2 hooks = hooks fs = fs.error == null ? fs : null resumedPart = true }
731. let boxed = boxOf(meta, meta.deploy == true, null)
732. let p = runClaude({ label = 'work' folder = meta.folder inFile = promptFile outFile = outFile errFile = errFile bin = meta.claude args = args timeout = meta.timeoutSeconds env = portEnv(meta.ports) session = meta.session ports = meta.ports sandbox = boxed }, (x) => {
733. finish(w, x)
734. return null
735. })
736. hooks.onProc(p, 'worker')
737. console.log('work: RESUMED ' + meta.ticket + ' session ' + meta.session + ' — ' + meta.claude + ' -p --resume ' + meta.claudeSession + ' (park ' + k + ') in ' + meta.folder + ' · pid ' + p.pid + ' · ports ' + meta.ports)
738. return true
739. }
741. // `check <run dir> [--post]`: the controller on a finished `work` run (a valid report.json) — or its
742. // verdict.json when there is one (`--again` runs the controller anew) — then, with --post, posting
743. // by the verdict. For runs made without --post, or when the controller / the post failed.
744. static runCheck = (argv, whenDone) => {
745. let done = whenDone == null ? noop : whenDone
746. let pos = positionals(argv)
747. if (pos.length != 2) {
748. console.log('usage: check <runs/session dir> [--post] [--again] [--controller-model M] [--controller-max-turns N] [--controller-timeout S] [--max-fails N]')
749. return false
750. }
751. let dir = fromCwd(pos[1])
752. while (dir.endsWith('/')) { dir = dir.slice(0, dir.length - 1) }
753. let post = argv.includes('--post')
754. let cs = controllerSettings(argv)
755. let cr = controllerRefusal(cs)
756. if (cr != null) {
757. console.log('check: REFUSED — ' + cr)
758. return false
759. }
760. let sj = readJson(dir + '/session.json')
761. if (sj.error != null) {
762. console.log('check: REFUSED — ' + sj.error + ' (a runs/<session> folder of `work`)')
763. return false
764. }
765. let meta = sj.value
766. if (meta.runDir == null) { meta.runDir = dir }
767. if (meta.ports == null) { meta.ports = setting(argv, '--ports', 'COLONY_PORTS', '8700-8749') }
768. let rep = reportFile(dir + '/report.json', false, { ticket = meta.ticket session = meta.session })
769. if (!rep.valid) {
770. console.log('check: REFUSED — no valid report in ' + dir)
771. return false
772. }
773. if (post) {
774. let tk = readToken()
775. if (tk.error != null) {
776. console.log('check: REFUSED — ' + tk.error + ' (--post writes to tickets)')
777. return false
778. }
779. }
780. let report = readJson(dir + '/report.json').value
781. let x = { ref = meta.ticket dir = dir post = post meta = meta report = report cs = cs label = 'CHECK' }
782. if (exists(dir + '/verdict.json') && !argv.includes('--again')) {
783. console.log('check: using the verdict of ' + dir + '/verdict.json (--again runs the controller anew)')
784. let cv = loadVerdict(dir, meta.ticket, meta.session)
785. afterVerdict(x, cv, done)
786. return true
787. }
788. let boxed = boxOf(meta, false, [dir])
789. runController({ dir = dir folder = meta.folder ref = meta.ticket session = meta.session host = meta.host ports = meta.ports bin = cs.bin model = cs.model maxTurns = cs.maxTurns mode = cs.mode timeout = cs.timeout budget = cs.budget sandbox = boxed }, meta, (cv) => {
790. afterVerdict(x, cv, done)
791. return null
792. })
793. return true
794. }
796. // mission 025: the controller of a salvaged run re-runs nothing that writes (the run is over; its folder may be in use)
797. static SALVAGE_NOTE = 'This is a SALVAGED run: the worker ran out of steps / time, and its report was written afterwards by a resumed session told to stop working. Re-run NOTHING that writes a file or starts a server or test (not even a gate): only read — `cat`, `ls`, `grep`, `diff`, `test -f` — and compare with the worker session. The working directory must not change. A claim whose command you may not re-run: check it in the worker session (the run files) instead.'
799. // `salvage <runs/session dir> [--post] [--finalize-max-turns N] [--finalize-timeout S] [--ports A-B] [controller options]`
800. // (mission 025): the finalize step for a FINISHED run whose worker ran out of steps / time without a report — resume
801. // its Claude session once, then the controller, then (with --post) post by the verdict like `check`.
802. // Mission 026: salvage takes a LEASE like `work` (ticket must be `open` → `in progress` + colony-lease marker + heartbeat),
803. // so `run` cannot pick the ticket meanwhile; no valid report → the lease goes back (`open`) like work's give-back;
804. // without --post the ticket stays leased (`./colony check <dir> --post` posts later) — exactly like `work`.
805. static runSalvage = (argv, whenDone) => {
806. let userDone = whenDone == null ? noop : whenDone
807. let done = (r) => {
808. console.log('SALVAGE COST: finalize ' + (r.finalizeCost == null ? '$?' : '$' + r.finalizeCost) + ' + controller ' + (r.controllerCost == null ? '–' : '$' + r.controllerCost) + ' (the worker before: $' + r.workerBefore + '; whole session now $' + round4((r.workerCost == null ? 0 : r.workerCost) + (r.controllerCost == null ? 0 : r.controllerCost)) + ')')
809. console.log('SALVAGE END — ' + r.ref + ': ' + r.outcome)
810. userDone(r)
811. return null
812. }
813. let pos = positionals(argv)
814. if (pos.length != 2) {
815. console.log('usage: salvage <runs/session dir> [--post] [--finalize-max-turns N] [--finalize-timeout S] [--ports A-B] [--controller-model M] [--controller-max-turns N] [--controller-timeout S] [--max-fails N]')
816. return false
817. }
818. let dir = fromCwd(pos[1])
819. while (dir.endsWith('/')) { dir = dir.slice(0, dir.length - 1) }
820. let post = argv.includes('--post')
821. let cs = controllerSettings(argv)
822. let cr = controllerRefusal(cs)
823. if (cr != null) {
824. console.log('salvage: REFUSED — ' + cr)
825. return false
826. }
827. let sj = readJson(dir + '/session.json')
828. if (sj.error != null) {
829. console.log('salvage: REFUSED — ' + sj.error + ' (a runs/<session> folder of `work`)')
830. return false
831. }
832. let meta = sj.value
833. meta.runDir = dir
834. let fs = finalizeSettings(argv, meta.timeoutSeconds)
835. if (fs.error != null) {
836. console.log('salvage: REFUSED — ' + fs.error)
837. return false
838. }
839. if (fs.maxTurns == '0') {
840. console.log('salvage: REFUSED — --finalize-max-turns 0 means no finalize step')
841. return false
842. }
843. if (meta.finalize != null && meta.finalize.why == REFUSED_WHY) {
844. // mission 027: its report was refused and the fix step ran — the worker did not run out of steps or time
845. console.log('salvage: REFUSED — the worker of ' + dir + ' did not run out of steps or time (outcome: ' + meta.outcome + ') — nothing to salvage')
846. return false
847. }
848. if (meta.finalize != null) {
849. console.log('salvage: REFUSED — ' + dir + ' had its finalize step already (' + meta.finalize.outcome + '); one finalize per session. Its report (if any): ./colony check ' + dir + (post ? ' --post' : ''))
850. return false
851. }
852. if (meta.parked != null && (meta.outcome == null || meta.outcome.startsWith('parked'))) {
853. console.log('salvage: REFUSED — ' + dir + ' is parked (' + meta.outcome + '); `run` resumes it')
854. return false
855. }
856. let outFile = dir + '/claude-result.json'
857. let rc = readClaude(outFile, meta.exit, false, meta.timeoutSeconds, 'worker')
858. let timedOut = meta.outcome != null && meta.outcome.includes('timed out after')
859. let why = finalizeWhy(rc.res, timedOut)
860. if (why == null) {
861. console.log('salvage: REFUSED — the worker of ' + dir + ' did not run out of steps or time (outcome: ' + meta.outcome + ') — nothing to salvage')
862. return false
863. }
864. if (exists(dir + '/report.json')) {
865. console.log('salvage: REFUSED — ' + dir + ' has a report.json already — use ./colony check ' + dir + (post ? ' --post' : ''))
866. return false
867. }
868. let host = hostName()
869. if (meta.host == null || meta.host.toLowerCase() != host.toLowerCase()) {
870. console.log('salvage: REFUSED — session ' + meta.session + ' ran on ' + meta.host + ', this host is ' + host + ' (its Claude session lives there)')
871. return false
872. }
873. if (meta.claudeSession == null || meta.folder == null || !exists(meta.folder)) {
874. console.log('salvage: REFUSED — session.json names no Claude session / no existing folder')
875. return false
876. }
877. let tref = parseRef(meta.ticket)
878. let ls = leaseSettings(argv)
879. if (ls.error != null) {
880. console.log('salvage: REFUSED — ' + ls.error)
881. return false
882. }
883. let tk = readToken()
884. if (tk.error != null) {
885. console.log('salvage: REFUSED — ' + tk.error + ' (salvage leases the ticket, so it always needs the token)')
886. return false
887. }
888. let token = tk.token
889. let got = getJson(ticketPath(tref.project, tref.number))
890. if (got.status != 200) {
891. console.log('salvage: REFUSED — ticket ' + meta.ticket + ' → ' + got.status + ' at ' + baseUrl())
892. return false
893. }
894. let st = got.json.ticket.state
895. // mission 030: a question ticket is never work — no salvage on one either
896. let qwhy = questionWhy(got.json.ticket)
897. if (qwhy != null) {
898. console.log('salvage: REFUSED — ' + meta.ticket + ' is ' + qwhy + ' (state ' + st + '); never work — nothing started')
899. return false
900. }
901. if (st != 'open') {
902. console.log('salvage: REFUSED — ' + meta.ticket + ' is "' + st + '" at ' + baseUrl() + ', not open' + (st == 'in progress' ? ' (another session holds it)' : '') + '; nothing started')
903. return false
904. }
905. let pt = option(argv, '--ports')
906. if (pt != null && pt != '') { meta.ports = pt }
907. meta.post = post
908. meta.salvaged = timestamp(now())
909. meta.leaseTtl = ls.ttl
910. meta.heartbeatSeconds = ls.heartbeat
911. let reason = meta.outcome == null ? why : meta.outcome
912. if (reason.startsWith('no valid report: ')) { reason = reason.slice(17) }
913. console.log('salvage: ' + meta.ticket + ' session ' + meta.session + ' — the worker ' + why + ' (' + reason + ', worker cost $' + meta.costUsd + ')' + (post ? ' · posts to ' + baseUrl() : ' · NOT posting (no --post)'))
914. let w = { meta = meta dir = dir outFile = outFile errFile = dir + '/claude-stderr.log' reportPath = dir + '/report.json' post = post ref = meta.ticket session = meta.session host = host project = tref.project number = tref.number token = token ticket = null mode = meta.permissionMode cs = cs done = done hooks = noHooks() fs = fs salvage = true label = 'SALVAGE' controllerNote = SALVAGE_NOTE }
915. let workerBefore = meta.costUsd
916. // mission 026: the LEASE (like work): open → in progress with the colony-lease marker; the heartbeat renews it
917. let expiry = now() + ls.ttl * 1000
918. let lease = postJson(ticketPath(tref.project, tref.number) + '/state', { state = 'in progress' text = 'A worker started on this ticket: it writes the report of an earlier session.' + NL + NL + leaseLine(meta.session, host, 'salvage', expiry) }, token)
919. if (lease.status != 201) {
920. console.log('salvage: FAILED — the lease (' + meta.ticket + ' → in progress) was refused: ' + lease.status + ' ' + lease.text + ' — nothing started')
921. return false
922. }
923. console.log('salvage: ' + meta.ticket + ' leased until ' + timestamp(expiry) + ' (heartbeat every ' + ls.heartbeat + ' s)')
924. let hb = startHeartbeat({ project = tref.project number = tref.number ref = meta.ticket session = meta.session host = host ttl = ls.ttl heartbeat = ls.heartbeat })
925. w.done = (r) => {
926. hb.stop()
927. r.ref = meta.ticket
928. // the finalize part's cost from the written session.json (the local meta is a stale copy by now — hybriel#38)
929. let now2 = readJson(dir + '/session.json')
930. r.finalizeCost = now2.error != null || now2.value.finalize == null ? null : now2.value.finalize.costUsd
931. r.workerBefore = workerBefore
932. done(r)
933. return null
934. }
935. startFinalize(w, meta, why, reason, null)
936. return true
937. }
939. // `cycle [--post] [--dry-run] [work options…]`: next → work → controller → post, ONCE, for the oldest
940. // eligible ticket whose dev folder is on this host; prints each step and the summed cost.
941. static runCycle = (argv, pick) => {
942. let host = hostName()
943. let p = pick(host)
944. if (p.error != null) {
945. console.log('CYCLE — ' + p.error)
946. return false
947. }
948. if (p.ticket == null) {
949. console.log('CYCLE — nothing to do: no eligible open ticket with its dev folder on ' + host)
950. return false
951. }
952. let t = p.ticket
953. let ref = t.project + t.ref
954. console.log('cycle: 1 next → ' + ref + ' "' + t.subject + '" (oldest eligible on ' + host + ')')
955. let num = '' + t.number
956. let wargv = ['work' t.project num]
957. let i = 1
958. while (i < argv.length) {
959. wargv.push(argv[i])
960. i = i + 1
961. }
962. console.log('cycle: 2 work → 3 controller → 4 ' + (argv.includes('--post') ? 'post by the verdict' : 'no post (no --post)'))
963. let started = runWork(wargv, (r) => {
964. let wc = r.workerCost == null ? 0 : r.workerCost
965. let cc = r.controllerCost == null ? 0 : r.controllerCost
966. console.log('CYCLE COST: $' + round4(wc + cc) + ' (worker ' + (r.workerCost == null ? '$?' : '$' + r.workerCost) + ' + controller ' + (r.controllerCost == null ? '–' : '$' + r.controllerCost) + ')')
967. console.log('CYCLE END — ' + ref + ': ' + r.outcome)
968. return null
969. })
970. if (!started) {
971. if (argv.includes('--dry-run')) { console.log('CYCLE DRY RUN — stopped before any Claude session; nothing written') }
972. else { console.log('CYCLE — no worker started for ' + ref + ' (see above)') }
973. }
974. return started
975. }