antcolony

42.4 KB

1. // daemon.hl — `run`: the scheduler as a long-running loop (mission 020, tickets antcolony#1/#2, build order 4
2. // part 2; concept docs/scheduler-agent.md §2 leases, §3 quota / parking / ports). Still ONE host, no WebSocket.
3. //
4. // Every --interval s (and once at the start) ONE iteration:
5. // 1. survey (next.hl) + the lease of every `in progress` ticket (lease.hl): an expired colony lease → the
6. // ticket back to `open` with a comment; a parked session of THIS host whose resume time is over → a resume
7. // candidate; a live lease → skipped (never a second session on a leased ticket, also after a restart);
8. // 2. candidates = due resumes first, then the eligible open tickets whose dev folder is on this host (oldest
9. // first) and not in use by a running session of this run;
10. // 3. free slots = --parallel − running; nothing to start or no slot → no quota probe;
11. // 4. quota: `claude -p /usage` (quota.hl, costs nothing) — at or above --reserve % of the 5 h window → no start;
12. // above --week-limit % (default 84, creator 2026-09-24) of the WEEKLY usage, or weekly unknown → no start;
13. // 100 turns either check off (--reserve 100: never blocks; --week-limit 100: never blocks, also when unknown);
14. // 5. start: a port range per session from --port-pool (--ports-per-session each, disjoint while running) →
15. // `work` (lease, worker, controller, post by the verdict) or `runResume`;
16. // 6. ONE line: `ITER <n> <time> · quota … · running … · started / resumed / expired … · skipped: …`.
17. // Mission 031: step 1 and the resume in 5 act on what the survey READ, seconds (or a librarian pass) earlier — so both
18. // re-read the ticket right before they write (lease.hl expireLease / stillLeased) and write nothing unless it is STILL
19. // `in progress` with the same lease; else `NOT expired …` / `NOT resumed …: re-read right before — <why> → nothing written`.
20. // A lease that has ENDED (the ticket left `in progress`, a `colony-deployed:` line, the session's own report …) is
21. // listed `… the lease of S has ended (…) → never expired`.
22. // A finished session prints `SESSION END <ref> <session> · <outcome> · cost …`.
23. // STOP: the `colony` wrapper turns SIGTERM/SIGINT into the file $COLONY_STOP_FILE ('finish' / 'park'); the run
24. // checks it twice a second. First signal: start nothing more, let the running sessions finish, then exit.
25. // Second signal: kill the running sessions' processes and PARK them (resumed by the next `run`). Never silent.
26. // `--once`: one iteration, wait for what it started, exit. Refuses the LIVE tickets server without `--live`.
27. // Mission 029 (antcolony#21): every iteration STARTS with the LIBRARIAN (librarian.hl), awaited: it reads the creator's new
28. // ticket texts and files decisions (no model call when nothing is new; quiet then); only afterwards the iteration picks and
29. // starts — a pass that failed or left a text unfiled starts nothing. `--librarian off` / COLONY_LIBRARIAN=off switches it
30. // off; `--librarian-model`, `--librarian-max` (texts per pass, 0 = all).
31. // Mission 034 (antcolony#25): with the librarian on, the QUOTA is read before it (`claude -p /usage`, $0): at or above the
32. // start threshold (a reached limit = 100 % always) → no librarian call, no start. A model call that still hits the usage
33. // limit (the librarian's, or a worker / controller parked on it) → `LIMIT:` pause until the reset (15 min when unknown):
34. // no librarian, no probe, no start; one line when it begins, one when it ends (README "Usage limit").
35. import { env } from 'hl:proc'
36. import { exists, readFile, mkDir } from 'hl:fs'
37. import { every, now, timestamp } from 'hl:time'
38. import { survey } from './next.hl'
39. import { baseUrl, readToken, getJson, ticketPath } from './tickets.hl'
40. import { runWork, runResume, hostName, runsDir } from './work.hl'
41. import { setting, isWholeNumber, round4 } from './claude.hl'
42. import { leaseOf, expireLease, stillLeased, leaseSettings } from './lease.hl'
43. import { probeUsage } from './quota.hl'
44. import { portRange, newSession } from './brief.hl'
45. import { librarianPass, librarianSettings } from './librarian.hl'
46. import { keywordPass } from './keywords.hl'
47. import { digestTick } from './status.hl'
48. import { questionWhy } from './relations.hl'
49. import { buildBrief, PORTS_MARKER } from './brief.hl'
50. import { readAgentToken } from './tickets.hl'
51. import { jsonErrorAt } from './jsoncheck.hl'
52. import { parseCapabilities, missingCapabilities } from './util.hl'
53. import AgentServer from './AgentServer.hl'
55. // options `run` hands to every `work` / resume unchanged
56. static PASS = ['--model' '--max-turns' '--permission-mode' '--timeout' '--max-budget-usd' '--claude' '--controller-claude' '--controller-model' '--controller-max-turns' '--controller-timeout' '--max-fails' '--lease-ttl' '--heartbeat' '--finalize-max-turns' '--finalize-timeout']
58. static LIVE_HOST = 'tickets.worldapi.org'
60. // mission 034: a usage limit without a known reset time pauses the librarian + starts this long, then tries once
61. static LIMIT_RETRY_MS = 900000
63. // the host part of a URL, lower case
64. static hostOf = (url) => {
65. let u = url.toLowerCase()
66. let i = u.indexOf('://')
67. if (i >= 0) { u = u.slice(i + 3) }
68. let j = u.indexOf('/')
69. if (j >= 0) { u = u.slice(0, j) }
70. let k = u.indexOf(':')
71. if (k >= 0) { u = u.slice(0, k) }
72. return u
73. }
75. static passOptions = (argv) => {
76. let out = []
77. let i = 0
78. while (i < argv.length) {
79. if (PASS.includes(argv[i]) && i + 1 < argv.length) {
80. out.push(argv[i])
81. out.push(argv[i + 1])
82. i = i + 2
83. } else { i = i + 1 }
84. }
85. return out
86. }
88. static money = (n) => {
89. if (n == null) { return '–' }
90. return '$' + round4(n)
91. }
93. static ago = (ms) => { return timestamp(ms).slice(0, 19) + 'Z' }
95. // `leases [--expire]`: every `in progress` ticket with its lease (a look at what `run` would do)
96. static runLeases = (argv) => {
97. let expire = argv.includes('--expire')
98. let tk = null
99. if (expire) {
100. if (hostOf(baseUrl()) == LIVE_HOST && !argv.includes('--live')) {
101. console.log('leases: REFUSED — ' + baseUrl() + ' is the LIVE tickets server; --expire writes, pass --live to do it there')
102. return false
103. }
104. tk = readToken()
105. if (tk.error != null) {
106. console.log('leases: REFUSED — ' + tk.error + ' (--expire writes)')
107. return false
108. }
109. }
110. let sv = survey()
111. if (sv.error != null) {
112. console.log('leases: ' + sv.error)
113. return false
114. }
115. let n = 0
116. let t0 = now()
117. for (e of sv.notEligible) {
118. let t = e.t
119. if (t.state == 'in progress') {
120. n = n + 1
121. let ref = t.project + t.ref
122. let d = getJson(ticketPath(t.project, t.number))
123. if (d.status != 200) { console.log('LEASE ' + ref + ' — cannot read: ' + d.status) } else {
124. let l = leaseOf(d.json.events)
125. if (l.kind == 'ready') { console.log('LEASE ' + ref + ' — built, waiting for the deploy (session ' + l.session + '): never expires — ./colony deployed ' + t.project + ' ' + t.number + ' after the deploy') }
126. else if (l.kind == 'ended') { console.log('LEASE ' + ref + ' — the lease of session ' + l.session + ' has ended (' + l.text + '): never expires') }
127. else if (l.kind != 'colony') { console.log('LEASE ' + ref + ' — no colony lease ("' + l.text + '"): never expires') }
128. else {
129. let st = l.parked ? 'parked (' + l.phase + ') until ' + ago(l.resumeMs) + ', ' : ''
130. if (t0 < l.expiryMs) { console.log('LEASE ' + ref + ' — session ' + l.session + ' on ' + l.host + ': ' + st + 'alive until ' + ago(l.expiryMs)) }
131. else if (!expire) { console.log('LEASE ' + ref + ' — session ' + l.session + ' on ' + l.host + ': ' + st + 'EXPIRED at ' + ago(l.expiryMs) + ' (--expire gives it back)') }
132. else {
133. let r = expireLease(t.project, t.number, l, tk.token)
134. let res = ' → back to open'
135. if (!r.ok && r.skipped == true) { res = ' → NOT expired, re-read right before: ' + r.text + ' (nothing written)' }
136. else if (!r.ok) { res = ' → giving it back FAILED: ' + r.text }
137. console.log('LEASE ' + ref + ' — session ' + l.session + ' on ' + l.host + ': ' + st + 'EXPIRED at ' + ago(l.expiryMs) + res)
138. }
139. }
140. }
141. }
142. }
143. console.log('LEASES: ' + n + ' ticket(s) in progress')
144. return true
145. }
147. static runRun = (argv) => {
148. let home = env('COLONY_HOME')
149. if (home == null || home == '') {
150. console.log('run: REFUSED — COLONY_HOME is not set (run through ./colony, which sets it)')
151. return false
152. }
153. let base = baseUrl()
154. let live = hostOf(base) == LIVE_HOST
155. if (live && !argv.includes('--live')) {
156. console.log('run: REFUSED — ' + base + ' is the LIVE tickets server (COLONY_TICKETS_URL unset or pointing there); pass --live to run against it')
157. return false
158. }
159. let capText = setting(argv, '--capabilities', 'COLONY_CAPABILITIES', '')
160. let hostCaps = parseCapabilities(capText)
161. let intervalText = setting(argv, '--interval', 'COLONY_RUN_INTERVAL', '60')
162. let parallelText = setting(argv, '--parallel', 'COLONY_PARALLEL', '1')
163. let reserveText = setting(argv, '--reserve', 'COLONY_QUOTA_RESERVE', '70')
164. let weekText = setting(argv, '--week-limit', 'COLONY_WEEK_LIMIT', '84')
165. let poolText = setting(argv, '--port-pool', 'COLONY_PORT_POOL', '8700-8799')
166. let perText = setting(argv, '--ports-per-session', 'COLONY_PORTS_PER_SESSION', '10')
167. let claudeBin = setting(argv, '--claude', 'COLONY_CLAUDE', 'claude')
168. let quotaBin = setting(argv, '--quota-claude', 'COLONY_QUOTA_CLAUDE', claudeBin)
169. if (!isWholeNumber(intervalText) || intervalText == '0') {
170. console.log('run: REFUSED — --interval must be whole seconds ≥ 1, got ' + intervalText)
171. return false
172. }
173. if (!isWholeNumber(parallelText) || parallelText == '0') {
174. console.log('run: REFUSED — --parallel must be a whole number ≥ 1, got ' + parallelText)
175. return false
176. }
177. if (!isWholeNumber(reserveText) || toNumber(reserveText) > 100) {
178. console.log('run: REFUSED — --reserve must be a percentage 0–100, got ' + reserveText)
179. return false
180. }
181. if (!isWholeNumber(weekText) || toNumber(weekText) > 100) {
182. console.log('run: REFUSED — --week-limit must be a percentage 0–100, got ' + weekText)
183. return false
184. }
185. // mission 035 (antcolony#14): `--serve PORT` = the scheduler proper — it starts no worker itself; agents connect to it
186. let serveText = setting(argv, '--serve', 'COLONY_SERVE', '')
187. let serveHost = setting(argv, '--serve-host', 'COLONY_SERVE_HOST', '127.0.0.1')
188. let ttlText = setting(argv, '--agent-ttl', 'COLONY_AGENT_TTL', '30')
189. let settleText = setting(argv, '--account-settle', 'COLONY_ACCOUNT_SETTLE', '60')
190. let serving = serveText != ''
191. if (serving && (!isWholeNumber(serveText) || serveText == '0' || toNumber(serveText) > 65535)) {
192. console.log('run: REFUSED — --serve must be a port 1–65535, got ' + serveText)
193. return false
194. }
195. if (serving && (!isWholeNumber(ttlText) || ttlText == '0')) {
196. console.log('run: REFUSED — --agent-ttl must be whole seconds ≥ 1, got ' + ttlText)
197. return false
198. }
199. if (serving && (!isWholeNumber(settleText))) {
200. console.log('run: REFUSED — --account-settle must be whole seconds ≥ 0, got ' + settleText)
201. return false
202. }
203. if (serving && argv.includes('--once')) {
204. console.log('run: REFUSED — --serve and --once do not go together (an agent needs the scheduler to stay)')
205. return false
206. }
207. let agentToken = { token = '' }
208. if (serving) {
209. agentToken = readAgentToken()
210. if (agentToken.error != null) {
211. console.log('run: REFUSED — ' + agentToken.error + ' (agents must authenticate)')
212. return false
213. }
214. }
215. let pool = portRange(poolText)
216. if (pool == null) {
217. console.log('run: REFUSED — --port-pool must be A-B (1 ≤ A ≤ B ≤ 65535), got ' + poolText)
218. return false
219. }
220. if (!isWholeNumber(perText) || perText == '0') {
221. console.log('run: REFUSED — --ports-per-session must be a whole number ≥ 1, got ' + perText)
222. return false
223. }
224. let interval = toNumber(intervalText)
225. let parallel = toNumber(parallelText)
226. let reserve = toNumber(reserveText)
227. let weekLimit = toNumber(weekText)
228. let per = toNumber(perText)
229. let size = pool.to - pool.from + 1
230. let slots = (size - size % per) / per
231. if (!serving && slots < parallel) {
232. console.log('run: REFUSED — the port pool ' + poolText + ' has ' + slots + ' range(s) of ' + per + ' ports, --parallel ' + parallel + ' needs ' + parallel)
233. return false
234. }
235. let libOn = setting(argv, '--librarian', 'COLONY_LIBRARIAN', 'on')
236. if (libOn != 'on' && libOn != 'off') {
237. console.log('run: REFUSED — --librarian must be on or off, got ' + libOn)
238. return false
239. }
240. let libSet = librarianSettings(argv, true)
241. if (libSet.error != null) {
242. console.log('run: REFUSED — ' + libSet.error)
243. return false
244. }
245. let ls = leaseSettings(argv)
246. if (ls.error != null) {
247. console.log('run: REFUSED — ' + ls.error)
248. return false
249. }
250. let tk = readToken()
251. if (tk.error != null) {
252. console.log('run: REFUSED — ' + tk.error + ' (run leases and posts)')
253. return false
254. }
255. let host = hostName()
256. let runs = runsDir(home)
257. mkDir(runs, 448)
258. let stopFile = env('COLONY_STOP_FILE')
259. let once = argv.includes('--once')
260. let pass = passOptions(argv)
262. let running = []
263. let live = {}
264. let libBusy = false
265. let busy = false
266. let stopping = null
267. let finished = false
268. let iter = 0
269. let loop = null
270. let watch = null
272. console.log('RUN: started on ' + host + (serving ? ' · SCHEDULER for agents on ' + serveHost + ':' + serveText + ' (starts no worker itself; agents offline after ' + ttlText + ' s without a sync)' : '') + ' · tickets ' + base + (live ? ' (LIVE)' : '') + ' · ' + (once ? 'once' : 'every ' + interval + ' s') + ' · parallel ' + parallel + ' · quota reserve ' + reserve + '% of the 5 h window, no start above ' + weekLimit + '% of the week (probe: ' + quotaBin + ' -p /usage) · ports ' + pool.from + '-' + pool.to + ', ' + per + ' per session · lease ttl ' + ls.ttl + ' s, heartbeat ' + ls.heartbeat + ' s · runs ' + runs + '/ · librarian ' + (libOn == 'on' ? 'on, first in every iteration (' + libSet.model + (libSet.max > 0 ? ', at most ' + libSet.max + ' text(s) per pass' : '') + ')' : 'off') + ' · stop file ' + (stopFile == null ? '— (no wrapper: SIGTERM kills without parking)' : stopFile))
274. // mission 034 (antcolony#25): the Claude usage limit. `limitUntil` (epoch ms) = "limited until": set when a model call
275. // of this run hit the limit — the librarian's (its pass reports `limited`), or a worker / controller that was parked on
276. // it. Until then: no librarian pass, no quota probe, no start (resumes too); leases are still looked after; the
277. // iterations are silent. ONE line when the pause begins, ONE when it ends. Unknown reset time → 15 min, then try once
278. // (the quota is read first — below).
279. let limitUntil = null
280. let limitReset = (resetMs) => { return resetMs == null ? now() + LIMIT_RETRY_MS : resetMs }
281. let pauseForLimit = (resetMs, why) => {
282. let until = limitReset(resetMs)
283. if (limitUntil == null) {
284. limitUntil = until
285. console.log('LIMIT: usage limit reached — librarian and starts paused until ' + ago(until) + (resetMs == null ? ' (no reset time known → ' + (LIMIT_RETRY_MS / 60000) + ' min, then try once)' : '') + ' · ' + why)
286. } else if (until > limitUntil) {
287. limitUntil = until
288. console.log('LIMIT: the pause is extended until ' + ago(until) + ' · ' + why)
289. }
290. return null
291. }
292. // the quota reading at or above the start threshold → why (text), else null. A REACHED limit (100 %, 5 h or week)
293. // always counts, also with --reserve 100 / --week-limit 100. An unknown week or a failed probe → null (the librarian
294. // runs; starts are still held by iterate as before).
295. let quotaBlock = (q) => {
296. if (q == null || q.error != null || q.percent == null) { return null }
297. if (q.percent >= 100) { return 'the 5 h limit is reached (100%)' }
298. if (q.weekly != null && q.weekly >= 100) { return 'the weekly limit is reached (100%)' }
299. if (reserve < 100 && q.percent >= reserve) { return '5 h ' + q.percent + '% ≥ reserve ' + reserve + '%' }
300. if (weekLimit < 100 && q.weekly != null && q.weekly > weekLimit) { return 'week ' + q.weekly + '% > week limit ' + weekLimit + '%' }
301. return null
302. }
303. let quotaText = (q) => { return 'quota ' + q.percent + '% of 5 h' + (q.resetsMs == null ? '' : ' (resets ' + ago(q.resetsMs) + ')') + (q.weekly == null ? '' : ', week ' + q.weekly + '%') + (q.cost != null && q.cost > 0 ? ', PROBE COST $' + q.cost : '') }
305. // mission 035 (antcolony#14): the agents that are connected (in memory only — an agent's next sync rebuilds all of it).
306. // agent = { host, seen, parallel, accepting, why, online }; `running` holds the sessions of the agents (entry.remote =
307. // the agent's host): what an agent reports as running, plus a command queued / just sent that the agent has not shown yet.
308. let agents = []
309. let seenEvents = []
310. let kwSeen = {}
311. let digestState = { day = '' }
312. let agentTtl = toNumber(ttlText) * 1000
313. // antcolony#16: the quota belongs to the Claude ACCOUNT. Agents on one account (`--account`) report the same reading; the
314. // scheduler pools it: freshest reading of the account, a limit pause of any of them, and one new start per account at a
315. // time until a reading taken AFTER that start (--account-settle s later) shows what it costs. accountStart = { account: ms }.
316. let settleMs = toNumber(settleText) * 1000
317. let accountStart = {}
318. let accountHosts = (acct) => {
319. let hs = []
320. for (a of agents) { if (agentOnline(a) && a.account == acct) { hs.push(a.host) } }
321. return hs
322. }
323. let accountReading = (acct) => {
324. let best = null
325. for (a of agents) {
326. if (agentOnline(a) && a.account == acct && a.quota != null && a.quota.at != null && (best == null || a.quota.at > best.at)) { best = a.quota }
327. }
328. return best
329. }
330. let accountBlock = (acct) => {
331. let hs = accountHosts(acct)
332. for (a of agents) {
333. if (agentOnline(a) && a.account == acct && a.limitUntil != null && a.limitUntil > now()) { return 'usage limit reached until ' + ago(a.limitUntil) + ' (hit on ' + a.host + ')' }
334. }
335. let q = accountReading(acct)
336. if (q == null) { return 'the quota is not read yet' }
337. if (now() - q.at > 300000) { return 'the newest quota reading is older than 5 min' }
338. if (q.percent >= 100) { return '5 h limit reached (100%)' }
339. if (q.weekly != null && q.weekly >= 100) { return 'weekly limit reached (100%)' }
340. if (reserve < 100 && q.percent >= reserve) { return 'quota 5 h ' + q.percent + '% ≥ reserve ' + reserve + '%' }
341. if (weekLimit < 100 && (q.weekly == null || q.weekly > weekLimit)) { return q.weekly == null ? 'week UNKNOWN' : 'week ' + q.weekly + '% > week limit ' + weekLimit + '%' }
342. let st = accountStart[acct]
343. if (st != null && q.at < st + settleMs) { return 'a start on it (' + ago(st) + ') is not in the quota reading yet' + (hs.length > 1 ? ' — shared by ' + hs.join(', ') : '') }
344. return null
345. }
346. let agentOf = (h) => {
347. if (h == null) { return null }
348. for (a of agents) { if (a.host.toLowerCase() == h.toLowerCase()) { return a } }
349. return null
350. }
351. let agentOnline = (a) => { return a != null && now() - a.seen <= agentTtl }
352. let hostRunning = (h) => {
353. let n = 0
354. for (r of running) { if (r.remote != null && r.remote.toLowerCase() == h.toLowerCase()) { n = n + 1 } }
355. return n
356. }
357. // the sum of what the connected agents can run at once (the "/N" of the ITER line)
358. let capacity = () => {
359. if (!serving) { return parallel }
360. let n = 0
361. for (a of agents) { if (agentOnline(a)) { n = n + a.parallel } }
362. return n
363. }
364. let dropHost = (h) => {
365. let out = []
366. for (x of running) { if (x.remote == null || x.remote.toLowerCase() != h.toLowerCase()) { out.push(x) } }
367. running = out
368. return null
369. }
370. let answer = (status, body) => { return { status = status body = body } }
371. let handleAgent = (method, path, body, auth) => {
372. if (path != '/agent/sync' || method != 'POST') { return answer(404, { error = 'POST /agent/sync only' }) }
373. if (auth == null || auth != 'Bearer ' + agentToken.token) { return answer(401, { error = 'the agent token is wrong or missing' }) }
374. if (body == null || body == '' || jsonErrorAt(body) >= 0) { return answer(400, { error = 'the body is not valid JSON' }) }
375. let j = JSON.parse(body)
376. if (j.host == null || hlTypeName(j.host) != 'String' || j.host == '' || j.running == null || j.events == null || j.parallel == null) { return answer(400, { error = 'host, parallel, running and events are required' }) }
377. let a = agentOf(j.host)
378. if (a == null) {
379. agents.push({ host = j.host account = j.host quota = null limitUntil = null caps = [] seen = now() parallel = j.parallel accepting = j.accepting why = j.why announced = false wasAccepting = false })
380. a = agentOf(j.host)
381. }
382. let wasOnline = agentOnline(a) && a.announced == true
383. a.seen = now()
384. a.caps = j.capabilities == null ? [] : parseCapabilities(j.capabilities.join(','))
385. a.parallel = j.parallel
386. a.account = j.account == null || j.account == '' ? j.host : j.account
387. a.quota = j.quota
388. a.limitUntil = j.limitUntil
389. a.accepting = j.accepting == true
390. a.why = j.why
391. if (!wasOnline) {
392. a.announced = true
393. console.log('AGENT: ' + j.host + ' connected · offers ' + (a.caps.length == 0 ? '—' : a.caps.join(', ')) + ' · parallel ' + j.parallel + ' · Claude account ' + a.account + ' · ' + (a.accepting ? 'accepting work' : 'not accepting work (' + j.why + ')') + ' · running ' + j.running.length + ' · ' + j.events.length + ' buffered event(s)')
394. } else if (a.accepting != a.wasAccepting) {
395. console.log('AGENT: ' + j.host + (a.accepting ? ' accepts work again' : ' does not accept work: ' + j.why))
396. }
397. a.wasAccepting = a.accepting
398. let ack = []
399. for (e of j.events) {
400. ack.push(e.id)
401. if (seenEvents.includes(e.id)) { }
402. else {
403. seenEvents.push(e.id)
404. if (seenEvents.length > 300) { seenEvents = seenEvents.slice(100) }
405. if (e.kind == 'ended') { console.log('SESSION END ' + e.ref + ' ' + e.session + ' · ' + e.outcome + ' · cost ' + money(e.workerCost + e.controllerCost) + ' (worker ' + money(e.workerCost) + ' + controller ' + money(e.controllerCost) + ') · on ' + j.host + ' · ports ' + e.ports) }
406. else if (e.kind == 'refused') { console.log('AGENT: ' + j.host + ' refused ' + e.ref + ' (' + e.session + ') — ' + e.why) }
407. }
408. }
409. // rebuild this host's entries: what it runs + a command it has not shown yet (sent one round ago and still missing = lost / refused)
410. let shown = []
411. let commands = []
412. let mine = []
413. for (x of running) { if (x.remote != null && x.remote.toLowerCase() == j.host.toLowerCase()) { mine.push(x) } }
414. dropHost(j.host)
415. for (r of j.running) {
416. shown.push(r.session)
417. running.push({ ref = r.ref session = r.session folder = r.folder slot = -1 ports = r.ports phase = r.phase remote = j.host proc = null sentAt = null cmd = null })
418. }
419. for (x of mine) {
420. if (shown.includes(x.session)) { }
421. else if (x.sentAt != null || x.cmd == null) { }
422. else {
423. x.sentAt = now()
424. commands.push(x.cmd)
425. running.push(x)
426. }
427. }
428. return answer(200, { ack = ack commands = commands })
429. }
430. let server = null
431. if (serving) { server = new AgentServer(port = toNumber(serveText), host = serveHost, handler = handleAgent) }
433. let slotFree = (i) => {
434. for (r of running) { if (r.slot == i) { return false } }
435. return true
436. }
437. let freeSlot = () => {
438. let i = 0
439. while (i < slots) {
440. if (slotFree(i)) { return i }
441. i = i + 1
442. }
443. return -1
444. }
445. let rangeOf = (i) => {
446. let a = pool.from + i * per
447. return a + '-' + (a + per - 1)
448. }
449. let runningSession = (s) => {
450. for (r of running) { if (r.session == s) { return true } }
451. return false
452. }
453. let folderBusy = (f) => {
454. for (r of running) { if (r.folder == f) { return true } }
455. return false
456. }
458. let checkEnd = () => {
459. if (finished || busy || libBusy || (running.length > 0 && !serving)) { return null }
460. if (stopping == null && !(once && iter >= 1)) { return null }
461. finished = true
462. if (loop != null) { loop.stop() }
463. if (watch != null) { watch.stop() }
464. console.log('RUN STOPPED — ' + (stopping != null ? 'stop requested (' + stopping + ')' : '--once') + ', ' + iter + ' iteration(s), ' + (serving ? running.length + ' session(s) keep running on their agents (leases in tickets)' : 'no session running'))
465. return null
466. }
468. let sessionEnded = (entry, r) => {
469. let out = []
470. for (x of running) { if (x.session != entry.session) { out.push(x) } }
471. running = out
472. let total = (r.workerCost == null ? 0 : r.workerCost) + (r.controllerCost == null ? 0 : r.controllerCost)
473. console.log('SESSION END ' + entry.ref + ' ' + entry.session + ' · ' + r.outcome + ' · cost ' + money(total) + ' (worker ' + money(r.workerCost) + ' + controller ' + money(r.controllerCost) + ') · ports ' + entry.ports + ' freed · running ' + running.length + '/' + capacity())
474. // mission 034: parked on the usage limit → no librarian call and no start into it until the reset
475. if (r.limited == true) { pauseForLimit(r.resetMs, entry.ref + ' ' + entry.session + ' was parked on it (' + r.limitText + ')') }
476. checkEnd()
477. return null
478. }
480. let startOne = (c) => {
481. let slot = freeSlot()
482. if (slot < 0) { return 'no port range free for ' + c.ref }
483. let ports = rangeOf(slot)
484. let entry = { ref = c.ref session = c.session folder = c.folder slot = slot ports = ports proc = null phase = 'worker' parkReason = null }
485. // antcolony#28: process, phase and park reason live in `live[session]`, NOT on `entry` — `running` is rebuilt by
486. // copying its entries (a Hybriel push copies), so a park reason set on a `running` entry never reached the hook
487. live[c.session] = { proc = null phase = 'worker' parkReason = null }
488. let hooks = {
489. onProc = (p, phase) => {
490. live[c.session].proc = p
491. live[c.session].phase = phase
492. return null
493. }
494. parkReason = () => { return live[c.session].parkReason }
495. }
496. // mission 031: a resume acts on what the survey saw — re-read first; still the same parked lease, or nothing is written
497. if (c.kind == 'resume') {
498. let chk = stillLeased('resume', c.project, c.number, c.lease)
499. if (!chk.ok) {
500. console.log('lease: NOT resumed ' + c.ref + ' (session ' + c.session + '): re-read right before — ' + chk.why + ' → nothing written')
501. return 'NOT resumed ' + c.ref + ' (' + c.session + '): re-read right before — ' + chk.why + ' → nothing written'
502. }
503. }
504. running.push(entry)
505. let ok = false
506. if (c.kind == 'resume') {
507. ok = runResume(runs + '/' + c.session, pass, (r) => {
508. sessionEnded(entry, r)
509. return null
510. }, hooks, ports)
511. } else {
512. let wargv = ['work' c.project '' + c.number '--post' '--session' c.session '--ports' ports]
513. for (o of pass) { wargv.push(o) }
514. ok = runWork(wargv, (r) => {
515. sessionEnded(entry, r)
516. return null
517. }, hooks)
518. }
519. if (!ok) {
520. let out = []
521. for (x of running) { if (x.session != entry.session) { out.push(x) } }
522. running = out
523. return c.ref + ' not started (see above)'
524. }
525. return (c.kind == 'resume' ? 'resumed ' : 'started ') + c.ref + ' (' + c.session + ', ports ' + ports + ')'
526. }
528. let iterate = null
529. let runLibrarian = null
530. let tick = () => {
531. if (busy || finished || stopping != null) { return null }
532. if (once && iter >= 1) { return null }
533. busy = true
534. iter = iter + 1
535. let head = 'ITER ' + iter + ' ' + ago(now())
536. // antcolony#17: the creator's /confirm /reject /prio /hold in comments — code only, no quota, before anything is picked
537. let kw = keywordPass(true, kwSeen)
538. if (kw == null || kw.error != null) { console.log(head + ' · keywords: ' + (kw == null ? 'failed' : kw.error)) }
539. // antcolony#19: the daily summary of what waits for the creator — code only, once a day after COLONY_DIGEST_HOUR
540. let dg = digestTick(digestState, now())
541. if (dg != null) { console.log(head + ' · ' + dg) }
542. // mission 034: a known limit → no model call (no librarian), no probe, no start until it resets
543. if (limitUntil != null) {
544. if (now() < limitUntil) {
545. iterate(head, 'usage limit', null, true, null)
546. return null
547. }
548. console.log('LIMIT: resumed — the pause until ' + ago(limitUntil) + ' is over; the quota is read first, then the librarian and starts')
549. limitUntil = null
550. }
551. if (libOn != 'on') {
552. iterate(head, null, null, false, null)
553. return null
554. }
555. // mission 034 (creator: "it cant recognize when it gets launches when we already hit the limit"): the quota FIRST
556. // (`claude -p /usage`, local, $0) — at or above the start threshold → no librarian call and no start this iteration
557. probeUsage(quotaBin, runs, (q) => {
558. let qb = quotaBlock(q)
559. if (qb != null) {
560. iterate(head, null, q, false, qb)
561. return null
562. }
563. runLibrarian(head, q)
564. return null
565. })
566. return null
567. }
569. // mission 029 — ORDER: the librarian pass comes FIRST and is awaited; only then the iteration looks at tickets and
570. // starts anything, so every brief built in this iteration has every creator decision written before it began.
571. // A pass that failed or left a creator text unfiled → nothing starts in this iteration (`hold`).
572. runLibrarian = (head, q) => {
573. libBusy = true
574. librarianPass({ argv = argv post = true auto = true stop = () => { return stopping != null } }, (s) => {
575. libBusy = false
576. // mission 034: the librarian's call hit the usage limit → pause (librarian + starts) until the reset
577. if (s != null && s.limited == true) {
578. pauseForLimit(s.resetMs, 'the librarian\'s model call hit it (' + s.limitText + '); ' + s.left + ' creator text(s) not filed yet')
579. iterate(head, 'usage limit', null, true, null)
580. return null
581. }
582. let hold = null
583. if (s == null || s.error != null) { hold = 'librarian failed (' + (s == null ? '?' : s.error) + ') → no start this iteration' }
584. else if (s.left != null && s.left > 0) { hold = 'librarian: ' + s.left + ' creator text(s) not filed yet → no start this iteration' }
585. // the reading is re-taken before a start when the pass made model calls (it may be minutes old then)
586. iterate(head, hold, s != null && s.calls != null && s.calls > 0 ? null : q, false, null)
587. return null
588. })
589. return null
590. }
592. // the rest of one iteration (after the librarian): leases, candidates, quota, starts.
593. // q = the quota reading of this iteration (null → probed here when something could start); paused = a known usage
594. // limit (mission 034): leases only, no start, no line unless a lease was expired; libSkipped = why the librarian did
595. // not run (the quota at or above the start threshold)
596. iterate = (head, hold, q0, paused, libSkipped) => {
597. if (stopping != null) {
598. console.log(head + ' · stop requested — nothing started · running ' + running.length + '/' + capacity())
599. busy = false
600. checkEnd()
601. return null
602. }
603. let notes = []
604. let did = []
605. // mission 035: an agent that has not synced for --agent-ttl s is offline — its entries go; its sessions keep their
606. // leases in tickets (the heartbeat is theirs) and are handled like any lease: renewed, or expired
607. for (a of agents) {
608. if (a.announced == true && !agentOnline(a)) {
609. a.announced = false
610. let had = hostRunning(a.host)
611. dropHost(a.host)
612. console.log('AGENT: ' + a.host + ' is OFFLINE (no sync for ' + ttlText + ' s) — ' + had + ' session(s) were on it; their leases in tickets decide (renewed by the workers, else expired)')
613. }
614. }
615. let sv = survey()
616. if (sv.error != null) {
617. console.log(head + ' · ERROR ' + sv.error + ' · running ' + running.length + '/' + capacity())
618. busy = false
619. checkEnd()
620. return null
621. }
622. let t0 = now()
623. let resumes = []
624. for (e of sv.notEligible) {
625. let t = e.t
626. if (t.state == 'in progress') {
627. let ref = t.project + t.ref
628. let d = getJson(ticketPath(t.project, t.number))
629. if (d.status != 200) { notes.push(ref + ' unreadable (' + d.status + ')') } else {
630. let l = leaseOf(d.json.events)
631. if (l.kind == 'ready') { notes.push(ref + ' built, waiting for the deploy') }
632. else if (l.kind == 'ended') { notes.push(ref + ' in progress, but the lease of ' + l.session + ' has ended (' + l.text + ') → never expired') }
633. else if (l.kind != 'colony') { notes.push(ref + ' in progress without a colony lease') }
634. else if (runningSession(l.session)) { }
635. else if (t0 >= l.expiryMs) {
636. // mission 031: expireLease re-reads the ticket first and writes nothing unless it is STILL this dead lease
637. let r = expireLease(t.project, t.number, l, tk.token)
638. if (r.ok) { did.push('expired ' + ref + ' (session ' + l.session + ' on ' + l.host + ', lease ended ' + ago(l.expiryMs) + ') → open') }
639. else if (r.skipped == true) {
640. did.push('NOT expired ' + ref + ' (session ' + l.session + ', lease ended ' + ago(l.expiryMs) + '): re-read right before — ' + r.text + ' → nothing written')
641. console.log('lease: NOT expired ' + ref + ' (session ' + l.session + '): re-read right before — ' + r.text + ' → nothing written')
642. }
643. else { did.push('EXPIRING ' + ref + ' FAILED: ' + r.text) }
644. } else if (l.parked) {
645. // mission 030: a question ticket is never work — a session parked on one (pre-030) is not resumed
646. if (questionWhy(t) != null) { notes.push(ref + ' parked, but ' + questionWhy(t) + ' → not resumed') }
647. else if (!serving && l.host.toLowerCase() != host.toLowerCase()) { notes.push(ref + ' parked on ' + l.host) }
648. else if (t0 < l.resumeMs) { notes.push(ref + ' parked until ' + ago(l.resumeMs)) }
649. else if (serving && !agentOnline(agentOf(l.host))) { notes.push(ref + ' parked on ' + l.host + ' (no agent online there)') }
650. else if (!serving && !exists(runs + '/' + l.session + '/session.json')) { notes.push(ref + ' parked, but ' + runs + '/' + l.session + ' is missing') }
651. else {
652. let sj = serving ? null : runs + '/' + l.session + '/session.json'
653. resumes.push({ kind = 'resume' ref = ref project = t.project number = t.number lease = l session = l.session folder = null sj = sj host = l.host })
654. }
655. } else { notes.push(ref + ' leased by ' + l.session + ' on ' + l.host + ' until ' + ago(l.expiryMs)) }
656. }
657. }
658. }
659. let cands = []
660. for (r of resumes) {
661. if (serving) { cands.push(r) }
662. else {
663. let m = JSON.parse(readFile(r.sj))
664. r.folder = m.folder
665. if (folderBusy(r.folder)) { notes.push(r.ref + ' (resume) dev folder busy') } else { cands.push(r) }
666. }
667. }
668. let other = 0
669. for (t of sv.eligible) {
670. let m = sv.reg.projects[t.project]
671. let ref = t.project + t.ref
672. if (m.dev == null || m.dev.host == null) { other = other + 1 }
673. else if (serving && !agentOnline(agentOf(m.dev.host))) { notes.push(ref + ' waits — no agent online on ' + m.dev.host) }
674. else if (!serving && m.dev.host.toLowerCase() != host.toLowerCase()) { other = other + 1 }
675. else if (missingCapabilities(m.needList, serving ? agentOf(m.dev.host).caps : hostCaps).length > 0) { notes.push(ref + ' waits — ' + m.dev.host + ' does not offer: ' + missingCapabilities(m.needList, serving ? agentOf(m.dev.host).caps : hostCaps).join(', ')) }
676. else if (m.dev.folder == null || folderBusy(m.dev.folder)) { notes.push(ref + ' dev folder busy') }
677. else {
678. let dup = null
679. for (c of cands) { if (dup == null && c.folder == m.dev.folder) { dup = c.ref } }
680. if (dup != null) { notes.push(ref + ' waits (same dev folder as ' + dup + ')') }
681. else { cands.push({ kind = 'new' ref = ref project = t.project number = t.number folder = m.dev.folder session = null host = m.dev.host }) }
682. }
683. }
684. if (other > 0) { notes.push(other + ' eligible on other hosts') }
685. let free = serving ? capacity() - running.length : parallel - running.length
686. let libNote = libSkipped == null ? [] : ['librarian not run: quota at or above the start threshold (' + libSkipped + ')']
687. let line = (quota, list) => {
688. let parts = [head, quota, 'running ' + running.length + '/' + capacity()]
689. for (x of did) { parts.push(x) }
690. for (x of list) { parts.push(x) }
691. for (x of libNote) { parts.push(x) }
692. if (notes.length > 0) { parts.push('skipped: ' + notes.join(', ')) }
693. console.log(parts.join(' · '))
694. return null
695. }
696. // mission 034: paused on a known usage limit — nothing starts; the iteration is silent unless a lease was acted on
697. if (paused == true) {
698. if (did.length > 0) { line('usage limit — librarian and starts paused until ' + ago(limitUntil), []) }
699. busy = false
700. checkEnd()
701. return null
702. }
703. if (hold != null && cands.length > 0) {
704. line(hold, [])
705. busy = false
706. checkEnd()
707. return null
708. }
709. // mission 035 (antcolony#14): the scheduler starts nothing itself — a candidate becomes a command for the agent of its
710. // dev host (one with a free slot that says it is accepting work; the quota is the agent's), sent with the agent's next sync
711. if (serving) {
712. let queued = []
713. for (c of cands) {
714. let a = agentOf(c.host)
715. if (!agentOnline(a)) { notes.push(c.ref + ' waits — no agent online on ' + c.host) }
716. else if (!a.accepting) { notes.push(c.ref + ' waits — agent ' + a.host + ' is not accepting work (' + a.why + ')') }
717. else if (hostRunning(a.host) >= a.parallel) { notes.push(c.ref + ' waits — agent ' + a.host + ' has all ' + a.parallel + ' slot(s) busy') }
718. else if (accountBlock(a.account) != null) { notes.push(c.ref + ' waits — Claude account ' + a.account + ': ' + accountBlock(a.account)) }
719. else if (c.kind == 'resume') {
720. // mission 031: a resume acts on what the survey saw — re-read first
721. let chk = stillLeased('resume', c.project, c.number, c.lease)
722. if (!chk.ok) {
723. console.log('lease: NOT resumed ' + c.ref + ' (session ' + c.session + '): re-read right before — ' + chk.why + ' → nothing written')
724. notes.push('NOT resumed ' + c.ref + ' (' + c.session + '): re-read right before — ' + chk.why + ' → nothing written')
725. } else {
726. running.push({ ref = c.ref session = c.session folder = null slot = -1 ports = '' phase = 'worker' remote = a.host proc = null sentAt = null cmd = { kind = 'resume' project = c.project number = c.number session = c.session folder = null brief = null } })
727. accountStart[a.account] = now()
728. queued.push('resume ' + c.ref + ' (' + c.session + ') → ' + a.host)
729. }
730. } else {
731. let session = newSession()
732. let b = buildBrief(c.project, '' + c.number, ['--session' session '--ports' PORTS_MARKER])
733. if (b.error != null) { notes.push(c.ref + ' not started: ' + b.error) }
734. else {
735. running.push({ ref = c.ref session = session folder = c.folder slot = -1 ports = '' phase = 'worker' remote = a.host proc = null sentAt = null cmd = { kind = 'start' project = c.project number = c.number session = session folder = c.folder brief = b.text } })
736. accountStart[a.account] = now()
737. queued.push('start ' + c.ref + ' (' + session + ') → ' + a.host)
738. }
739. }
740. }
741. let names = []
742. for (a of agents) { if (agentOnline(a)) { names.push(a.host + ' ' + hostRunning(a.host) + '/' + a.parallel + (a.accepting ? '' : ' (not accepting: ' + a.why + ')')) } }
743. let accts = []
744. for (a of agents) {
745. if (agentOnline(a) && !accts.includes(a.account)) {
746. accts.push(a.account)
747. let q = accountReading(a.account)
748. names.push('account ' + a.account + (q == null ? ' quota unknown' : ' ' + q.percent + '%' + (q.weekly == null ? '' : ' week ' + q.weekly + '%')) + ' [' + accountHosts(a.account).join('+') + ']')
749. }
750. }
751. line('agents: ' + (names.length == 0 ? 'none online' : names.join(', ')), queued)
752. busy = false
753. checkEnd()
754. return null
755. }
756. if (cands.length == 0 || free <= 0) {
757. let why = cands.length == 0 ? 'nothing to start' : 'all ' + parallel + ' slot(s) busy, waiting: ' + cands.length
758. if (q0 == null) { line('quota not read (' + why + ')', []) }
759. else if (q0.error != null) { line('quota UNKNOWN (' + q0.error + ') · ' + why, []) }
760. else { line(quotaText(q0) + (libSkipped == null ? '' : ' → librarian and starts paused') + ' · ' + why, []) }
761. busy = false
762. checkEnd()
763. return null
764. }
765. let withQuota = (fn) => {
766. if (q0 != null) { return fn(q0) }
767. return probeUsage(quotaBin, runs, fn)
768. }
769. withQuota((q) => {
770. if (stopping != null) {
771. line('stop requested — nothing started', [])
772. busy = false
773. checkEnd()
774. return null
775. }
776. if (q.error != null) {
777. line('quota UNKNOWN (' + q.error + ') → no start', [])
778. busy = false
779. checkEnd()
780. return null
781. }
782. let qt = quotaText(q)
783. // --reserve 100 = the 5 h window never blocks (creator 2026-09-24: first live runs up to 100 %)
784. if (reserve < 100 && q.percent >= reserve) {
785. let names = []
786. for (c of cands) { names.push(c.ref) }
787. line(qt + ' ≥ reserve ' + reserve + '% → no start (waiting: ' + names.join(', ') + ')', [])
788. busy = false
789. checkEnd()
790. return null
791. }
792. // mission 023 (creator): no new start (resumes too) while the WEEKLY usage is above the week limit (84 %);
793. // a probe without a weekly number starts nothing either — better wait than overspend the week;
794. // --week-limit 100 = the week never blocks (also when unknown)
795. if (weekLimit < 100 && (q.weekly == null || q.weekly > weekLimit)) {
796. let wn = []
797. for (c of cands) { wn.push(c.ref) }
798. line(qt + (q.weekly == null ? ', week UNKNOWN' : ' > week limit ' + weekLimit + '%') + ' → no start (waiting: ' + wn.join(', ') + ')', [])
799. busy = false
800. checkEnd()
801. return null
802. }
803. // mission 034: a REACHED limit (100 %) holds every start, also with --reserve 100 / --week-limit 100
804. let reached = quotaBlock(q)
805. if (reached != null) {
806. let rn = []
807. for (c of cands) { rn.push(c.ref) }
808. line(qt + ' → ' + reached + ' → no start (waiting: ' + rn.join(', ') + ')', [])
809. busy = false
810. checkEnd()
811. return null
812. }
813. let started = []
814. let i = 0
815. while (i < cands.length && running.length < parallel) {
816. let c = cands[i]
817. if (c.session == null) { c.session = newSession() }
818. started.push(startOne(c))
819. i = i + 1
820. }
821. while (i < cands.length) {
822. notes.push(cands[i].ref + ' waits for a slot')
823. i = i + 1
824. }
825. line(qt + (reserve >= 100 ? ' · 5 h reserve off (100%)' : ' < reserve ' + reserve + '%') + (weekLimit >= 100 ? ' · week limit off (100%)' : ''), started)
826. busy = false
827. checkEnd()
828. return null
829. })
830. return null
831. }
833. let checkStop = () => {
834. if (finished || stopFile == null || stopFile == '' || !exists(stopFile)) { return null }
835. let mode = readFile(stopFile).trim()
836. if (stopping == null) {
837. stopping = 'finish'
838. console.log('RUN: stop requested — starting nothing more; ' + running.length + ' running session(s) finish (a second SIGTERM/SIGINT parks them instead)')
839. }
840. if (mode == 'park' && stopping != 'park') {
841. stopping = 'park'
842. console.log('RUN: second stop signal — parking ' + running.length + ' running session(s)')
843. for (r of running) {
844. if (live[r.session] != null) {
845. live[r.session].parkReason = 'the scheduler run was stopped (second SIGTERM/SIGINT)'
846. if (live[r.session].proc != null) {
847. console.log('RUN: killing ' + live[r.session].phase + ' pid ' + live[r.session].proc.pid + ' of ' + r.ref + ' ' + r.session + ' → parked')
848. live[r.session].proc.kill()
849. }
850. }
851. }
852. }
853. checkEnd()
854. return null
855. }
857. watch = every(0.5)
858. on watch.tick(x) {
859. checkStop()
860. return null
861. }
862. loop = every(interval)
863. on loop.tick(x) {
864. tick()
865. return null
866. }
867. tick()
868. return true
869. }