antcolony

49.2 KB

1. // librarian.hl — `librarian`: the creator's decisions in one place per project (mission 029, ticket antcolony#21).
2. //
3. // Why: a worker sees its own ticket, the concept and the TITLES of other tickets — the creator's answers and decisions
4. // made in comments never reach it, so workers ask about things long decided (gitoria#3). Sources (creator, 2026-09-24:
5. // "the librarian should be enough. i have to adopt to the workflow"): TICKETS only, plus once the architect's seed of
6. // decisions made in chat (`--seed`). The librarian reads every text
7. // the CREATOR wrote in tickets (comments, state-change texts, ticket texts, edits — incl. answers on question tickets),
8. // asks ONE one-shot Claude step per new text (`--json-schema templates/librarian.schema.json`) whether it is a decision,
9. // for which project (or `all`), under which topic, what it means and which earlier entries it corrects, and keeps:
10. // - `<dev folder>/DECISIONS.md` per project (current state only, by topic, contents at the top, newest first per topic)
11. // + `DECISIONS-archive.md` beside it (superseded entries, kept with quote + link);
12. // dev folder not on this host → the code folder; neither → `librarian/projects/<name>/DECISIONS.md`;
13. // mission 030: a project with `"workers": false` (hybriel) ALWAYS → `librarian/projects/<name>/` (never into its repo);
14. // - `templates/decisions-global.md` (+ `decisions-global-archive.md`) for `all`.
15. // Every entry = date · the creator's EXACT words (taken from the ticket by the code — the model may only pick a
16. // verbatim excerpt, checked character for character) · where (ticket link / chat seed) · ONE line "means" marked as the
17. // librarian's (or, for the chat seed, the architect's) reading. Never a reading without the quote.
18. // Data: `librarian/decisions.json` (the entries — the .md files are rendered from it, never edited by hand),
19. // `librarian/state.json` (cursor = newest ticket `updatedMs` read, floor = --since, done = event id → outcome, tries),
20. // `librarian/calls/<event>-{prompt.md,result.json,stderr.log}` (every model call, for the audit). READS tickets only.
21. //
22. // ./colony librarian dry run: lists the new creator texts; no model call, nothing written
23. // ./colony librarian --post-files classify every new text (one model call each) + write the files
24. // [--since DATE|ms] [--max N] [--model M] [--retry-refused]
25. // ./colony librarian --post-files --seed FILE [--seed-source TEXT] import decisions made in CHAT (architect's seed file)
26. // ./colony librarian --render re-render every file from decisions.json (no tickets, no model)
27. // ORDER (the creator: a worker must know decisions from OTHER tickets written seconds earlier): `run` awaits one
28. // `librarianPass` at the start of EVERY iteration before it picks / starts anything, and starts nothing in an iteration
29. // that left a creator text unfiled (daemon.hl); `brief`, `work`, `cycle` by hand await one pass before the brief is built
30. // (`librarianFirst`, scheduler.hl). One GET /api/tickets + the updated tickets; no model call unless a creator text is
31. // new; files are rewritten only when something changed.
32. import { env } from 'hl:proc'
33. import { readFile, writeFile, exists, mkDir, remove } from 'hl:fs'
34. import { now, timestamp } from 'hl:time'
35. import { sha256 } from 'hl:crypto'
36. import { fetch } from 'hl:fetch'
37. import { jsonErrorAt } from './jsoncheck.hl'
38. import { loadRegistry } from './registry.hl'
39. import { getJson, baseUrl, isCreatorEvent } from './tickets.hl'
40. import { runClaude, readClaude, usageLine, setting, isWholeNumber, round4, rateLimitOf } from './claude.hl'
41. import { NL, readJson, fromCwd, isList, isObject, isString, option, sortByNumber, sortStrings, fenceFor } from './util.hl'
42. import { isoToMs } from './quota.hl'
44. static DEFAULT_CREATORS = 'Caramboleyo,creator'
45. static LOCK_STALE_MS = 900000
46. static FULL_LINES_DEFAULT = 150
47. static LAYOUTS_DEFAULT = '/media/STORAGE/projects/antcolony-docs/docs/layouts-conventions.md'
48. static GENERATED = '<!-- generated by the antcolony librarian (antcolony-scheduler: ./colony librarian) from librarian/decisions.json — do not edit by hand, it is rewritten -->'
50. // ---- settings --------------------------------------------------------------------------------------------------
51. static homeDir = () => {
52. let h = env('COLONY_HOME')
53. return h == null || h == '' ? null : h
54. }
56. // the librarian's own folder (state, entries, calls): COLONY_LIBRARIAN_DIR, else <scheduler>/librarian
57. static libDir = () => {
58. let d = env('COLONY_LIBRARIAN_DIR')
59. if (d != null && d != '') { return fromCwd(d) }
60. let h = homeDir()
61. return h == null ? './librarian' : h + '/librarian'
62. }
64. // the decisions for ALL projects: COLONY_DECISIONS_GLOBAL, else <scheduler>/templates/decisions-global.md
65. static globalFile = () => {
66. let g = env('COLONY_DECISIONS_GLOBAL')
67. if (g != null && g != '') { return fromCwd(g) }
68. let h = homeDir()
69. return h == null ? './templates/decisions-global.md' : h + '/templates/decisions-global.md'
70. }
72. // DECISIONS.md → DECISIONS-archive.md, decisions-global.md → decisions-global-archive.md
73. static archiveOf = (path) => { return path.endsWith('.md') ? path.slice(0, path.length - 3) + '-archive.md' : path + '-archive' }
75. static layoutsConventions = () => {
76. let v = env('COLONY_LAYOUTS_CONVENTIONS')
77. return v == null || v == '' ? LAYOUTS_DEFAULT : v
78. }
80. // the creator's display names in tickets (the API shows names, not user ids — see README "Librarian")
81. static creatorNames = () => {
82. let v = env('COLONY_CREATOR_NAMES')
83. if (v == null || v.trim() == '') { v = DEFAULT_CREATORS }
84. let out = []
85. for (p of v.split(',')) { if (p.trim() != '') { out.push(p.trim()) } }
86. return out
87. }
89. static thisHost = () => {
90. let h = env('COLONY_HOST')
91. if (h == null || h == '') { h = exists('/etc/hostname') ? readFile('/etc/hostname').trim() : '' }
92. return h
93. }
95. // → { dir, model, bin, timeout, maxTurns, max } or { error }. `pre` = option prefix ('' for `librarian`, '--librarian-'
96. // style names are used by `run`: --librarian-model, --librarian-claude, --librarian-max)
97. static librarianSettings = (argv, inRun) => {
98. let model = setting(argv, inRun ? '--librarian-model' : '--model', 'COLONY_LIBRARIAN_MODEL', 'sonnet')
99. let fallbackBin = setting(argv, '--claude', 'COLONY_CLAUDE', 'claude')
100. let bin = setting(argv, '--librarian-claude', 'COLONY_LIBRARIAN_CLAUDE', fallbackBin)
101. let timeoutText = setting(argv, '--librarian-timeout', 'COLONY_LIBRARIAN_TIMEOUT', '300')
102. let turnsText = setting(argv, '--librarian-max-turns', 'COLONY_LIBRARIAN_MAX_TURNS', '4')
103. let maxText = setting(argv, inRun ? '--librarian-max' : '--max', 'COLONY_LIBRARIAN_MAX', '0')
104. if (!isWholeNumber(timeoutText) || timeoutText == '0') { return { error = 'the librarian timeout must be whole seconds ≥ 1, got ' + timeoutText } }
105. if (!isWholeNumber(turnsText) || turnsText == '0') { return { error = 'the librarian max turns must be a whole number ≥ 1, got ' + turnsText } }
106. if (!isWholeNumber(maxText)) { return { error = (inRun ? '--librarian-max' : '--max') + ' must be a whole number (0 = no limit), got ' + maxText } }
107. // antcolony#20: `local` (or `local:<alias>`) = a small model on a local llama-server (OpenAI-compatible endpoint) answers
108. // first; an unreachable server or an answer the checks refuse goes on to Claude (`--librarian-fallback`, default sonnet, `off` = no fallback)
109. let localUrl = setting(argv, '--librarian-local-url', 'COLONY_LOCAL_URL', 'http://127.0.0.1:8080/v1')
110. let fallback = setting(argv, '--librarian-fallback', 'COLONY_LIBRARIAN_FALLBACK', 'sonnet')
111. let localTimeout = setting(argv, '--librarian-local-timeout', 'COLONY_LOCAL_TIMEOUT', '120')
112. if (!isWholeNumber(localTimeout) || localTimeout == '0') { return { error = 'the local model timeout must be whole seconds ≥ 1, got ' + localTimeout } }
113. return { dir = libDir() model = model bin = bin timeout = timeoutText maxTurns = turnsText max = toNumber(maxText) localUrl = localUrl fallback = fallback localTimeout = toNumber(localTimeout) }
114. }
116. // ---- small helpers ---------------------------------------------------------------------------------------------
117. static oneLine = (s) => {
118. let t = ('' + s).replaceAll('\r', ' ').replaceAll(NL, ' ')
119. while (t.indexOf(' ') >= 0) { t = t.replaceAll(' ', ' ') }
120. return t.trim()
121. }
122. static clip = (s, n) => {
123. let t = oneLine(s)
124. return t.length <= n ? t : t.slice(0, n - 1) + '…'
125. }
126. static money = (n) => { return n == null ? '$?' : '$' + round4(n) }
128. // `--since`: epoch ms, `YYYY-MM-DD`, `YYYY-MM-DDTHH:MM[:SS]` (UTC) → ms or null
129. static sinceMs = (s) => {
130. if (s == null || s == '') { return null }
131. if (isWholeNumber(s)) { return toNumber(s) }
132. let t = s
133. if (t.length == 10) { t = t + 'T00:00:00Z' }
134. else if (t.length == 16) { t = t + ':00Z' }
135. else if (t.length == 19) { t = t + 'Z' }
136. return isoToMs(t)
137. }
139. // ---- where the files live --------------------------------------------------------------------------------------
140. // → { path, where } — the project's DECISIONS.md: dev folder on THIS host, else its code folder on this host, else
141. // the librarian's own folder (a project registered on another host, or one without a registry file)
142. static projectFile = (name, reg) => {
143. if (name == 'all') { return { path = globalFile() where = 'all projects' } }
144. let host = thisHost().toLowerCase()
145. let m = reg == null ? null : reg.projects[name]
146. // mission 030: a project with `"workers": false` (hybriel: its own process in its own repo) never gets a file from us
147. if (m != null && !m.takesWorkers) { return { path = libDir() + '/projects/' + name + '/DECISIONS.md' where = 'librarian folder (the project takes no workers: "workers": false — nothing is written into its repo)' } }
148. if (m != null) {
149. if (m.dev != null && m.dev.host != null && m.dev.folder != null && m.dev.host.toLowerCase() == host && exists(m.dev.folder)) { return { path = m.dev.folder + '/DECISIONS.md' where = 'dev folder' } }
150. if (m.code != null && m.code.host != null && m.code.folder != null && m.code.host.toLowerCase() == host && exists(m.code.folder)) { return { path = m.code.folder + '/DECISIONS.md' where = 'code folder (the dev folder is on ' + m.dev.host + ')' } }
151. }
152. return { path = libDir() + '/projects/' + name + '/DECISIONS.md' where = 'librarian folder (' + (m == null ? 'no registry file' : 'no dev or code folder on this host') + ')' }
153. }
155. // ---- store + state ---------------------------------------------------------------------------------------------
156. static loadStore = (dir) => {
157. let f = dir + '/decisions.json'
158. if (!exists(f)) { return { entries = [] } }
159. let r = readJson(f)
160. if (r.error != null) { return { error = r.error } }
161. if (!isObject(r.value) || !isList(r.value.entries)) { return { error = f + ': no entries list' } }
162. return r.value
163. }
164. static saveStore = (dir, store) => {
165. writeFile(dir + '/decisions.json', JSON.stringify(store), 420)
166. return null
167. }
168. static loadState = (dir) => {
169. let f = dir + '/state.json'
170. if (!exists(f)) { return { cursorMs = 0 floorMs = 0 done = {} tries = {} passes = 0 } }
171. let r = readJson(f)
172. if (r.error != null) { return { error = r.error } }
173. let s = r.value
174. if (s.done == null) { s.done = {} }
175. if (s.tries == null) { s.tries = {} }
176. if (s.cursorMs == null) { s.cursorMs = 0 }
177. if (s.floorMs == null) { s.floorMs = 0 }
178. if (s.passes == null) { s.passes = 0 }
179. return s
180. }
181. static saveState = (dir, st) => {
182. writeFile(dir + '/state.json', JSON.stringify(st), 420)
183. return null
184. }
186. // ---- the creator's texts ---------------------------------------------------------------------------------------
187. // the creator's own words in one event → list of strings (empty = nothing to read: a bare state change, a link)
188. static quotesOf = (e, t) => {
189. let out = []
190. if (e.kind == 'created') {
191. if (t.subject != null && t.subject.trim() != '') { out.push(t.subject.trim()) }
192. if (t.summary != null && t.summary.trim() != '') { out.push(t.summary.trim()) }
193. } else if (e.kind == 'comment' || e.kind == 'state') {
194. if (e.text != null && e.text.trim() != '') { out.push(e.text.trim()) }
195. } else if (e.kind == 'edit') {
196. if (e.subjectChanged == true && e.newSubject != null && e.newSubject.trim() != '') { out.push(e.newSubject.trim()) }
197. if (e.summaryChanged == true && e.newSummary != null && e.newSummary.trim() != '') { out.push(e.newSummary.trim()) }
198. }
199. return out
200. }
202. static whatOf = (e) => {
203. if (e.kind == 'created') { return 'ticket text (opened the ticket)' }
204. if (e.kind == 'state') { return 'state change ' + e.from + ' → ' + e.to }
205. if (e.kind == 'edit') { return 'edit of the ticket' }
206. return 'comment'
207. }
209. static textOf = (e, t, evs, i, quotes) => {
210. let before = []
211. let j = i - 8
212. if (j < 0) { j = 0 }
213. while (j < i) {
214. let b = evs[j]
215. let bt = b.text == null ? '' : b.text
216. if (b.kind == 'edit') { bt = 'edited: ' + (b.newSubject == null ? '' : b.newSubject) }
217. before.push({ author = b.author kind = b.kind to = b.to == null ? '' : b.to when = b.when text = clip(bt, 600) })
218. j = j + 1
219. }
220. let refText = t.project + '#' + t.number
221. let urlText = baseUrl() + t.href
222. let subj = t.subject == null ? '' : t.subject
223. let summ = t.summary == null ? '' : t.summary
224. let whenText = e.when == null ? '' : e.when
225. return { key = e.id kind = e.kind what = whatOf(e) author = e.author when = whenText date = whenText.slice(0, 10) createdMs = e.createdMs quotes = quotes project = t.project number = t.number ref = refText url = urlText subject = subj summary = summ state = t.state isQuestion = subj.startsWith('Question') before = before }
226. }
228. // every new creator text → { texts (oldest first), cursorMs (newest updatedMs seen), read, total } or { error }
229. static gatherTexts = (names, floorMs, cursorMs, done) => {
230. let all = getJson('/api/tickets')
231. if (all.status != 200) { return { error = 'GET /api/tickets → ' + all.status + (all.text == null ? '' : ' ' + clip(all.text, 120)) } }
232. let texts = []
233. let maxUpd = cursorMs
234. let read = 0
235. let lowest = cursorMs > floorMs ? cursorMs : floorMs
236. for (t of all.json.tickets) {
237. let upd = t.updatedMs == null ? 0 : t.updatedMs
238. if (upd > maxUpd) { maxUpd = upd }
239. if (upd >= lowest) {
240. let d = getJson(t.apiHref)
241. if (d.status != 200) { return { error = 'GET ' + t.apiHref + ' → ' + d.status } }
242. read = read + 1
243. let evs = d.json.events
244. let tk = d.json.ticket
245. let i = 0
246. while (i < evs.length) {
247. let e = evs[i]
248. let ms = e.createdMs == null ? 0 : e.createdMs
249. if (isCreatorEvent(t.project, e, names) && done[e.id] == null && ms >= floorMs) {
250. let q = quotesOf(e, tk)
251. if (q.length > 0) { texts.push(textOf(e, tk, evs, i, q)) }
252. }
253. i = i + 1
254. }
255. }
256. }
257. sortByNumber(texts, (x) => { return x.createdMs })
258. return { texts = texts cursorMs = maxUpd read = read total = all.json.tickets.length }
259. }
261. // ---- topics ----------------------------------------------------------------------------------------------------
262. static normTopic = (s) => {
263. let t = oneLine(s).toLowerCase()
264. let p = t.indexOf(' (')
265. if (p > 0) { t = t.slice(0, p) }
266. return t.trim()
267. }
268. static topicsOf = (entries, project) => {
269. let out = []
270. for (e of entries) { if (e.status == 'current' && e.project == project && !out.includes(e.topic)) { out.push(e.topic) } }
271. sortStrings(out)
272. return out
273. }
275. // ---- the model call --------------------------------------------------------------------------------------------
276. static promptOf = (x, allowed, entries) => {
277. let topics = {}
278. for (p of allowed) {
279. let ts = topicsOf(entries, p)
280. if (ts.length > 0) { topics[p] = ts }
281. }
282. let cur = []
283. for (e of entries) {
284. if (e.status == 'current') { cur.push({ id = e.id project = e.project topic = e.topic quote = clip(e.quotes.join(' · '), 300) means = e.means }) }
285. }
286. let ct = { quotes = x.quotes what = x.what author = x.author when = x.when }
287. let input = { creatorText = ct }
288. input.ticket = { ref = x.ref project = x.project subject = x.subject summary = clip(x.summary, 2000) state = x.state isQuestion = x.isQuestion url = x.url }
289. input.historyBefore = x.before
290. input.allowedProjects = allowed
291. input.existingTopics = topics
292. input.currentEntries = cur
293. let js = JSON.stringify(input)
294. let f = fenceFor(js)
295. return readFile('./templates/librarian.md').trim() + NL + NL + '## Input (JSON)' + NL + NL + f + 'json' + NL + js + NL + f + NL
296. }
298. // antcolony#20: a small local model answers the same question (same prompt, the schema enforced by the server) →
299. // { answer } or { reason }; the call's files stay in <dir>/calls/ like Claude's
300. static localAnswer = (c, x, allowed, entries) => {
301. let base = c.dir + '/calls/' + x.key + '-local'
302. let prompt = promptOf(x, allowed, entries)
303. writeFile(base + '-prompt.md', prompt, 384)
304. let alias = c.model.indexOf(':') > 0 ? c.model.slice(c.model.indexOf(':') + 1) : 'local'
305. let schema = JSON.parse(readFile('./templates/librarian.schema.json'))
306. let body = { model = alias temperature = 0 max_tokens = 700 messages = [{ role = 'user' content = prompt }] response_format = { type = 'json_schema' json_schema = { name = 'librarian' strict = true schema = schema } } }
307. let res = fetch(c.localUrl + '/chat/completions', { method = 'POST' headers = { 'user-agent' = 'antcolony-scheduler' } json = body timeoutMs = c.localTimeout * 1000 })
308. if (res == null) { return { reason = 'the local model server ' + c.localUrl + ' did not answer' } }
309. let text = res.text()
310. writeFile(base + '-result.json', text == null ? '' : text, 384)
311. if (res.status != 200) { return { reason = 'the local model server answered ' + res.status } }
312. if (jsonErrorAt(text) >= 0) { return { reason = 'the local model server answer is not JSON' } }
313. let v = JSON.parse(text)
314. if (!isObject(v) || !isList(v.choices) || v.choices.length == 0 || !isObject(v.choices[0].message) || !isString(v.choices[0].message.content)) { return { reason = 'the local model answer has no message' } }
315. let content = v.choices[0].message.content
316. if (jsonErrorAt(content) >= 0) { return { reason = 'the local model answer is cut off or not JSON (finish: ' + v.choices[0].finish_reason + ')' } }
317. let a = JSON.parse(content)
318. if (!isObject(a)) { return { reason = 'the local model answer is not a JSON object' } }
319. return { answer = a }
320. }
322. // one text → done({ answer } | { reason }), with cost; the files of the call stay in <dir>/calls/
323. static classify = (c, x, allowed, entries, done) => {
324. let calls = c.dir + '/calls'
325. if (c.model == 'local' || c.model.startsWith('local:')) {
326. mkDir(calls, 448)
327. let la = localAnswer(c, x, allowed, entries)
328. let why = la.reason
329. if (la.answer != null) {
330. let chk = checkAnswer(la.answer, x, allowed, entries)
331. if (chk.problems == null) {
332. console.log(' LOCAL MODEL (' + c.model + ') answered')
333. done({ answer = la.answer cost = 0 })
334. return null
335. }
336. why = 'the answer failed the checks: ' + chk.problems.join('; ')
337. }
338. if (c.fallback == 'off' || c.fallback == '') {
339. done({ reason = 'local model: ' + why })
340. return null
341. }
342. console.log(' LOCAL MODEL could not answer (' + why + ') → Claude ' + c.fallback)
343. let c2 = c
344. c2.model = c.fallback
345. return classify(c2, x, allowed, entries, done)
346. }
347. mkDir(calls, 448)
348. let base = calls + '/' + x.key
349. let inFile = base + '-prompt.md'
350. let outFile = base + '-result.json'
351. let errFile = base + '-stderr.log'
352. writeFile(inFile, promptOf(x, allowed, entries), 384)
353. let args = ['-p' '--output-format' 'json' '--json-schema' readFile('./templates/librarian.schema.json') '--model' c.model '--max-turns' c.maxTurns '--permission-mode' 'default' '--permission-prompts' 'none' '--strict-mcp-config' '--tools' '' '--no-session-persistence']
354. let p = runClaude({ label = 'librarian' folder = c.dir inFile = inFile outFile = outFile errFile = errFile bin = c.bin args = args timeout = c.timeout env = null }, (r) => {
355. let rc = readClaude(outFile, r.ev.exit, r.timedOut, c.timeout, 'librarian')
356. let cost = null
357. if (rc.res != null) {
358. cost = rc.res.total_cost_usd
359. console.log(' ' + usageLine('LIBRARIAN USAGE', rc.res))
360. }
361. // mission 034 (antcolony#25): the call ran into the Claude usage limit (live: subtype "success", is_error, 429,
362. // empty answer) → reported as LIMITED, not as "no answer" (the pass stops; `run` pauses until the reset)
363. if (rc.value != null) {
364. let rl = rateLimitOf(rc.value, rc.res)
365. if (rl.limited) {
366. done({ limited = true resetMs = rl.resetMs limitText = rl.text cost = cost })
367. return null
368. }
369. }
370. if (rc.reason != null) {
371. done({ reason = rc.reason cost = cost })
372. return null
373. }
374. if (rc.res.structured_output == null) {
375. done({ reason = 'no structured answer (Claude: ' + rc.res.subtype + (rc.res.is_error ? ', error' : '') + ', ' + rc.res.num_turns + ' turns, exit ' + r.ev.exit + ')' cost = cost })
376. return null
377. }
378. done({ answer = rc.res.structured_output cost = cost })
379. return null
380. })
381. return p
382. }
384. // the code decides what is stored: → { problems } or { decision: false, why } or { decision: true, project, topic,
385. // quotes, quoteNote, means, supersedes, dropped, why }
386. static KEYS = ['decision' 'project' 'topic' 'quote' 'means' 'supersedes' 'why']
387. static checkAnswer = (a, x, allowed, entries) => {
388. let problems = []
389. if (!isObject(a)) { return { problems = ['the answer is not an object'] } }
390. for (k of a.keys()) { if (!KEYS.includes(k)) { problems.push("unknown key '" + k + "'") } }
391. for (k of KEYS) { if (a[k] == null) { problems.push("'" + k + "' is missing") } }
392. if (problems.length > 0) { return { problems = problems } }
393. if (hlTypeName(a.decision) != 'Boolean') { problems.push("'decision' must be true or false") }
394. if (!isString(a.why)) { problems.push("'why' must be text") }
395. if (problems.length > 0) { return { problems = problems } }
396. let why = oneLine(a.why)
397. if (a.decision == false) { return { decision = false why = why } }
398. if (!isString(a.project) || !allowed.includes(a.project)) { problems.push("'project' must be one of " + allowed.join(', ') + ", got '" + a.project + "'") }
399. let topic = isString(a.topic) ? normTopic(a.topic) : ''
400. if (topic == '' || topic.length > 40) { problems.push("'topic' must be 1–40 characters, got '" + a.topic + "'") }
401. let means = isString(a.means) ? oneLine(a.means) : ''
402. if (means == '') { problems.push("'means' must not be empty for a decision") }
403. if (!isString(a.quote)) { problems.push("'quote' must be text") }
404. if (!isList(a.supersedes)) { problems.push("'supersedes' must be a list") }
405. if (problems.length > 0) { return { problems = problems } }
406. // the quote: the whole text, or a VERBATIM excerpt of it (checked here — the model never writes the creator's words)
407. let quotes = x.quotes
408. let note = ''
409. let ex = a.quote.trim()
410. if (ex != '') {
411. let hit = false
412. for (q of x.quotes) { if (q.includes(ex)) { hit = true } }
413. if (hit) {
414. let whole = false
415. for (q of x.quotes) { if (q == ex) { whole = true } }
416. if (!whole) { quotes = [ex] }
417. } else { note = 'the excerpt the model gave is not verbatim — the whole text is stored' }
418. }
419. let sup = []
420. let dropped = []
421. for (id of a.supersedes) {
422. let found = false
423. for (e of entries) { if (e.id == id && e.status == 'current') { found = true } }
424. if (found && !sup.includes(id)) { sup.push(id) } else { dropped.push('' + id) }
425. }
426. // the reading is ONE line of at most 300 characters (first live pass: 5 of 155 answers were longer — clipped, not refused)
427. if (means.length > 300) {
428. means = means.slice(0, 299) + '…'
429. note = note == '' ? 'the reading was longer than 300 characters — clipped' : note + '; the reading was clipped to 300 characters'
430. }
431. return { decision = true project = a.project topic = topic quotes = quotes quoteNote = note means = means supersedes = sup dropped = dropped why = why }
432. }
434. // ---- rendering -------------------------------------------------------------------------------------------------
435. static slug = (s) => {
436. let out = ''
437. let low = s.toLowerCase()
438. let i = 0
439. while (i < low.length) {
440. let c = low.charCodeAt(i)
441. let ch = low.slice(i, i + 1)
442. if ((c >= 97 && c <= 122) || (c >= 48 && c <= 57) || c == 45 || c == 95) { out = out + ch }
443. else if (c == 32) { out = out + '-' }
444. i = i + 1
445. }
446. return out
447. }
449. static linkOf = (e) => {
450. if (e.source.kind == 'seed') { return 'source: ' + e.source.text }
451. let subj = clip(e.source.subject, 70).replaceAll('[', '(').replaceAll(']', ')')
452. return '[' + e.source.ref.replaceAll('#', ' #') + ' — ' + subj + '](' + e.source.url + ')'
453. }
455. static readingLabel = (e) => {
456. if (e.means == null || e.means == '') { return 'means: — (no reading recorded; the quote is all there is)' }
457. let who = e.readingBy == 'architect' ? "the architect's reading" : "the librarian's reading"
458. return 'means (' + who + ', not the creator\'s words): ' + e.means
459. }
461. static entryLines = (e, byId) => {
462. let o = []
463. let multi = false
464. for (q of e.quotes) { if (q.indexOf(NL) >= 0) { multi = true } }
465. let ctx = e.context != null && e.context != '' ? e.context + ' ' : ''
466. if (!multi) {
467. let qs = []
468. for (q of e.quotes) { qs.push('**"' + q + '"**') }
469. o.push('- ' + e.date + ' · ' + ctx + qs.join(' · ') + ' · ' + linkOf(e))
470. } else {
471. o.push('- ' + e.date + ' · ' + ctx + linkOf(e) + ' · the creator wrote:')
472. let first = true
473. for (q of e.quotes) {
474. if (!first) { o.push(' >') }
475. for (l of q.split(NL)) { o.push(' > ' + l.replaceAll('\r', '')) }
476. first = false
477. }
478. }
479. o.push(' - ' + readingLabel(e))
480. if (e.status == 'superseded') {
481. let by = byId[e.supersededBy]
482. let byText = by == null ? e.supersededBy : e.supersededBy + ' — **"' + clip(by.quotes.join(' · '), 120) + '"** · ' + linkOf(by)
483. o.push(' - superseded on ' + e.supersededOn + ' by ' + byText)
484. }
485. return o
486. }
488. // the text of one DECISIONS file (current or archive) for `name` from `entries` (already filtered)
489. static renderFile = (name, entries, archive, byId) => {
490. let o = []
491. let label = name == 'all' ? 'all projects' : name
492. o.push('# Decisions — ' + label + (archive ? ' (archive: superseded)' : ''))
493. o.push('')
494. o.push(GENERATED)
495. o.push('')
496. if (archive) {
497. o.push('Superseded decisions of the creator for **' + label + '** — kept with the quote and where it was said. The current ones are in `' + (name == 'all' ? 'decisions-global.md' : 'DECISIONS.md') + '` beside this file. Nothing here is valid any more.')
498. } else {
499. o.push('The creator\'s decisions for **' + label + '** — current state only (superseded ones: `' + (name == 'all' ? 'decisions-global-archive.md' : 'DECISIONS-archive.md') + '` beside this file).')
500. o.push('**A decision here overrides anything else; ask only if it is not decided here.**')
501. o.push('Each entry: date · the creator\'s exact words in quotes · where it was said · "means" = a READING (the librarian\'s or the architect\'s), not the creator\'s words — when in doubt, the quote counts.')
502. }
503. o.push('')
504. if (entries.length == 0) {
505. o.push(archive ? '_(nothing superseded yet)_' : '_(no decisions recorded yet)_')
506. o.push('')
507. return o.join(NL)
508. }
509. let topics = []
510. for (e of entries) { if (!topics.includes(e.topic)) { topics.push(e.topic) } }
511. sortStrings(topics)
512. o.push('## Contents')
513. for (tp of topics) {
514. let n = 0
515. for (e of entries) { if (e.topic == tp) { n = n + 1 } }
516. o.push('- [' + tp + '](#' + slug(tp) + ') — ' + n)
517. }
518. for (tp of topics) {
519. o.push('')
520. o.push('## ' + tp)
521. o.push('')
522. let list = []
523. for (e of entries) { if (e.topic == tp) { list.push(e) } }
524. sortByNumber(list, (e) => { return 0 - e.ms })
525. for (e of list) { for (l of entryLines(e, byId)) { o.push(l) } }
526. }
527. o.push('')
528. return o.join(NL)
529. }
531. // write every file whose text changed → list of 'path (n current, k superseded)' lines written
532. static renderAll = (store, reg) => {
533. let byId = {}
534. for (e of store.entries) { byId[e.id] = e }
535. let names = ['all']
536. for (n of reg.names) { names.push(n) }
537. for (e of store.entries) { if (!names.includes(e.project)) { names.push(e.project) } }
538. let written = []
539. for (name of names) {
540. let cur = []
541. let old = []
542. for (e of store.entries) {
543. if (e.project == name) { if (e.status == 'current') { cur.push(e) } else { old.push(e) } }
544. }
545. let pf = projectFile(name, reg)
546. let slash = pf.path.lastIndexOf('/')
547. if (slash > 0) { mkDir(pf.path.slice(0, slash), 493) }
548. let text = renderFile(name, cur, false, byId)
549. if (!exists(pf.path) || readFile(pf.path) != text) {
550. writeFile(pf.path, text, 420)
551. written.push(pf.path + ' (' + cur.length + ' current · ' + pf.where + ')')
552. }
553. let ap = archiveOf(pf.path)
554. if (old.length > 0 || exists(ap)) {
555. let at = renderFile(name, old, true, byId)
556. if (!exists(ap) || readFile(ap) != at) {
557. writeFile(ap, at, 420)
558. written.push(ap + ' (' + old.length + ' superseded)')
559. }
560. }
561. }
562. return written
563. }
565. // ---- the brief's part (brief.hl) -------------------------------------------------------------------------------
566. static STOP = ['that' 'this' 'with' 'from' 'have' 'should' 'would' 'will' 'into' 'when' 'what' 'which' 'there' 'their' 'then' 'than' 'also' 'only' 'make' 'made' 'like' 'just' 'some' 'each' 'every' 'more' 'most' 'other' 'about' 'after' 'before' 'worker' 'workers' 'creator' 'colony' 'must' 'need' 'needs' 'does' 'done' 'them' 'they' 'your' 'here' 'where' 'were' 'been' 'being' 'because' 'while' 'within' 'without' 'over' 'under' 'again' 'same' 'such' 'very' 'work' 'working' 'question' 'ticket' 'tickets' 'project' 'projects' 'reading' 'librarian' 'means' 'decision' 'decisions' 'should' 'these' 'those' 'could' 'into' 'onto' 'upon' 'still' 'already' 'later' 'first' 'next' 'used' 'uses' 'using']
568. // the words of a text that can match: lower case, letters/digits, ≥ 4 characters, no stop words → stems (first 5)
569. static stemsOf = (text) => {
570. let out = []
571. let low = ('' + text).toLowerCase()
572. let w = ''
573. let i = 0
574. while (i <= low.length) {
575. let c = i < low.length ? low.charCodeAt(i) : 32
576. if ((c >= 97 && c <= 122) || (c >= 48 && c <= 57)) { w = w + low.slice(i, i + 1) }
577. else {
578. if (w.length >= 4 && !STOP.includes(w)) {
579. let st = w.length > 5 ? w.slice(0, 5) : w
580. if (!out.includes(st)) { out.push(st) }
581. }
582. w = ''
583. }
584. i = i + 1
585. }
586. return out
587. }
589. static demote = (line) => { return line.startsWith('#') ? '##' + line : line }
591. // the lines of one decisions file for the brief: the whole file while it is short, else its contents + the topic
592. // sections whose name or reading lines share a word with the ticket (subject + summary)
593. static fileLines = (label, pf, ticketText, full) => {
594. let o = []
595. if (!exists(pf.path)) {
596. o.push('_(no decisions recorded yet for ' + label + ')_')
597. return o
598. }
599. let lines = readFile(pf.path).split(NL)
600. let body = []
601. for (l of lines) { if (!l.startsWith('# ') && !l.startsWith('<!-- generated')) { body.push(l) } }
602. while (body.length > 0 && body[0].trim() == '') { body = body.slice(1) }
603. if (lines.length <= full) {
604. for (l of body) { o.push(demote(l)) }
605. return o
606. }
607. // long file: split into the head (before the first topic) and the topic sections
608. let head = []
609. let sections = []
610. let cur = null
611. for (l of body) {
612. if (l.startsWith('## ') && l != '## Contents') {
613. if (cur != null) { sections.push(cur) }
614. cur = { name = l.slice(3).trim() lines = [l] }
615. } else if (cur != null) { cur.lines.push(l) } else { head.push(l) }
616. }
617. if (cur != null) { sections.push(cur) }
618. let want = stemsOf(ticketText)
619. let picked = []
620. let names = []
621. for (s of sections) {
622. let hay = s.name
623. for (l of s.lines) { if (l.indexOf('means (') >= 0) { hay = hay + ' ' + l.slice(l.indexOf('means (')) } }
624. let has = stemsOf(hay)
625. let hit = false
626. for (st of want) { if (has.includes(st)) { hit = true } }
627. if (hit) {
628. picked.push(s)
629. names.push(s.name)
630. }
631. }
632. for (l of head) { o.push(demote(l)) }
633. o.push('')
634. o.push('**This file is long (' + lines.length + ' lines): only the topics that share a word with this ticket are carried here (' + (names.length == 0 ? 'none matched' : names.join(', ')) + '; ' + picked.length + ' of ' + sections.length + ' topics). Search the whole file `' + pf.path + '` before you ask anything.**')
635. for (s of picked) {
636. o.push('')
637. for (l of s.lines) { o.push(demote(l)) }
638. }
639. return o
640. }
642. // the brief's section "Decisions of the creator" (after the conventions): global + the project's file, with paths
643. static decisionLines = (project, reg, ticketText) => {
644. let fullText = env('COLONY_DECISIONS_FULL_LINES')
645. let full = fullText != null && isWholeNumber(fullText) ? toNumber(fullText) : FULL_LINES_DEFAULT
646. let host = thisHost()
647. let g = projectFile('all', reg)
648. let p = projectFile(project, reg)
649. let o = []
650. o.push('## Decisions of the creator — a decision here overrides anything else; ask only if it is not decided here')
651. o.push('Collected by the librarian from the creator\'s own words (tickets; for all projects also the architect\'s chat seed). Each entry quotes the creator exactly; the "means" line is only a reading — when in doubt, the quote counts. Superseded decisions are in the `-archive.md` file beside each file.')
652. o.push('- All projects: `' + host + ':' + g.path + '`')
653. o.push('- ' + project + ': `' + host + ':' + p.path + '`')
654. o.push('- **Search both files (e.g. grep your topic) before you ask a question.** A question about something decided there is refused.')
655. o.push('')
656. o.push('### For all projects')
657. o.push('')
658. for (l of fileLines('all projects', g, ticketText, full)) { o.push(l) }
659. o.push('')
660. o.push('### For ' + project)
661. o.push('')
662. for (l of fileLines(project, p, ticketText, full)) { o.push(l) }
663. return o
664. }
666. // ---- seed (decisions made in chat) -----------------------------------------------------------------------------
667. // the architect's seed file: `## Topic` headings, `- "quote" · "quote" · reading: …` bullets (continuation lines
668. // indented). A bullet without a quoted part is not an entry (printed as skipped). → { items, skipped }
669. static parseSeed = (text) => {
670. let items = []
671. let skipped = []
672. let topic = 'general'
673. let cur = null
674. let flush = () => {
675. if (cur == null) { return null }
676. let item = cur
677. cur = null
678. let r = item.indexOf('reading:')
679. let head = r >= 0 ? item.slice(0, r) : item
680. let means = r >= 0 ? oneLine(item.slice(r + 8)) : ''
681. let quotes = []
682. let ctx = ''
683. let i = head.indexOf('"')
684. if (i > 0) { ctx = head.slice(0, i).trim() }
685. while (i >= 0) {
686. let j = head.indexOf('"', i + 1)
687. if (j < 0) { i = -1 } else {
688. let q = head.slice(i + 1, j)
689. if (q.trim() != '') { quotes.push(q) }
690. i = head.indexOf('"', j + 1)
691. }
692. }
693. if (quotes.length == 0) { skipped.push(topic + ': ' + clip(item, 100)) }
694. else { items.push({ topic = topic context = ctx quotes = quotes means = means }) }
695. return null
696. }
697. for (l of text.split(NL)) {
698. if (l.startsWith('## ')) {
699. flush()
700. topic = normTopic(l.slice(3))
701. } else if (l.startsWith('- ')) {
702. flush()
703. cur = l.slice(2).trim()
704. } else if (cur != null && l.startsWith(' ') && l.trim() != '') {
705. cur = cur + ' ' + l.trim()
706. } else { flush() }
707. }
708. flush()
709. return { items = items skipped = skipped }
710. }
712. static importSeed = (c, file, sourceText) => {
713. let p = fromCwd(file)
714. if (!exists(p)) { return { error = 'no such seed file: ' + p } }
715. let store = loadStore(c.dir)
716. if (store.error != null) { return { error = store.error } }
717. let st = loadState(c.dir)
718. if (st.error != null) { return { error = st.error } }
719. let ps = parseSeed(readFile(p))
720. let added = 0
721. let have = 0
722. let base = isoToMs('2026-09-24T00:00:00Z')
723. let dateText = '2026-09-23/24'
724. let src = sourceText
725. for (it of ps.items) {
726. let h = sha256(it.quotes.join(NL)).slice(0, 16)
727. let key = 'seed:' + h
728. if (st.done[key] != null) { have = have + 1 } else {
729. let eid = 'g-' + h.slice(0, 12)
730. let entry = { id = eid project = 'all' topic = it.topic quotes = it.quotes context = it.context means = it.means readingBy = 'architect' date = dateText ms = base source = { kind = 'seed' text = src file = p } added = now() status = 'current' }
731. store.entries.push(entry)
732. let d = st.done
733. d[key] = 'decision ' + eid
734. st.done = d
735. added = added + 1
736. console.log('LIBRARIAN SEED ' + eid + ' · all · ' + it.topic + ' · "' + clip(it.quotes.join(' · '), 100) + '"')
737. }
738. }
739. for (s of ps.skipped) { console.log('librarian: seed item without a quote — skipped (no reading without the quote): ' + s) }
740. saveStore(c.dir, store)
741. saveState(c.dir, st)
742. return { added = added have = have skipped = ps.skipped.length store = store }
743. }
745. // ---- the pass --------------------------------------------------------------------------------------------------
746. static lockFile = (dir) => { return dir + '/lock' }
747. static takeLock = (dir) => {
748. let f = lockFile(dir)
749. if (exists(f)) {
750. let r = readJson(f)
751. let ms = r.error == null && r.value.ms != null ? r.value.ms : 0
752. if (now() - ms < LOCK_STALE_MS) { return { error = 'another librarian pass holds ' + f + ' since ' + timestamp(ms).slice(0, 19) + 'Z (stale after 15 min)' } }
753. }
754. writeFile(f, JSON.stringify({ ms = now() }), 384)
755. return { ok = true }
756. }
757. static dropLock = (dir) => {
758. remove(lockFile(dir))
759. return null
760. }
762. // o = { argv, post (write), auto (called by `run`: quiet when nothing is new, files only when something changed),
763. // stop () → true when the run wants to stop }. done(summary) is called once:
764. // summary = { newTexts, decisions, not, refused, failed, cost, written, error, left } — mission 034: a call that ran into the
765. // Claude usage limit stops the pass → summary.limited = true, resetMs (null = unknown), limitText (no try counted)
766. static librarianPass = (o, done) => {
767. let argv = o.argv
768. let c = librarianSettings(argv, o.auto)
769. if (c.error != null) {
770. console.log('librarian: REFUSED — ' + c.error)
771. done({ error = c.error })
772. return false
773. }
774. if (homeDir() == null) {
775. console.log('librarian: REFUSED — COLONY_HOME is not set (run through ./colony, which sets it)')
776. done({ error = 'no COLONY_HOME' })
777. return false
778. }
779. mkDir(c.dir, 448)
780. let reg = loadRegistry()
781. for (e of reg.errors) { console.log('registry: ' + e) }
782. let allowed = []
783. for (n of reg.names) { allowed.push(n) }
785. // --render: files from the store, nothing else
786. if (argv.includes('--render')) {
787. let store = loadStore(c.dir)
788. if (store.error != null) {
789. console.log('librarian: ' + store.error)
790. done({ error = store.error })
791. return false
792. }
793. let w = renderAll(store, reg)
794. for (l of w) { console.log('LIBRARIAN FILE ' + l) }
795. console.log('LIBRARIAN RENDER — ' + w.length + ' file(s) written, the others unchanged')
796. done({ written = w })
797. return true
798. }
799. // --seed FILE: decisions made in chat (the architect's seed) — no model call
800. let seed = option(argv, '--seed')
801. if (seed != null) {
802. if (!o.post) {
803. console.log('librarian: REFUSED — --seed writes: add --post-files')
804. done({ error = 'seed without --post-files' })
805. return false
806. }
807. let lk = takeLock(c.dir)
808. if (lk.error != null) {
809. console.log('librarian: BUSY — ' + lk.error)
810. done({ error = lk.error })
811. return false
812. }
813. let srcText = option(argv, '--seed-source')
814. if (srcText == null || srcText == '') { srcText = 'architect session 2026-09-23/24, chat' }
815. let si = importSeed(c, seed, srcText)
816. if (si.error != null) {
817. dropLock(c.dir)
818. console.log('librarian: ' + si.error)
819. done({ error = si.error })
820. return false
821. }
822. let w = renderAll(si.store, reg)
823. for (l of w) { console.log('LIBRARIAN FILE ' + l) }
824. dropLock(c.dir)
825. console.log('LIBRARIAN SEED — ' + si.added + ' new entr' + (si.added == 1 ? 'y' : 'ies') + ', ' + si.have + ' already there (HAVE), ' + si.skipped + ' item(s) without a quote skipped · ' + w.length + ' file(s) written')
826. done({ decisions = si.added written = w })
827. return true
828. }
830. let st = loadState(c.dir)
831. if (st.error != null) {
832. console.log('librarian: ' + st.error)
833. done({ error = st.error })
834. return false
835. }
836. let sinceText = option(argv, '--since')
837. let floor = st.floorMs
838. if (sinceText != null) {
839. let s = sinceMs(sinceText)
840. if (s == null) {
841. console.log('librarian: REFUSED — --since must be epoch ms, YYYY-MM-DD or YYYY-MM-DDTHH:MM (UTC), got ' + sinceText)
842. done({ error = 'bad --since' })
843. return false
844. }
845. floor = s
846. }
847. // --redo EVENT[,EVENT…] (ticket event ids): classify these texts again — their outcome is forgotten and an entry the
848. // librarian made from them is removed (its own reading was wrong; not a correction by the creator, so no archive)
849. let redo = option(argv, '--redo')
850. if (redo != null) {
851. if (!o.post) {
852. console.log('librarian: REFUSED — --redo writes: add --post-files')
853. done({ error = 'redo without --post-files' })
854. return false
855. }
856. let rstore = loadStore(c.dir)
857. if (rstore.error != null) {
858. console.log('librarian: ' + rstore.error)
859. done({ error = rstore.error })
860. return false
861. }
862. let keys = []
863. for (k of redo.split(',')) { if (k.trim() != '') { keys.push(k.trim()) } }
864. let keepDone = {}
865. for (k of st.done.keys()) { if (!keys.includes(k)) { keepDone[k] = st.done[k] } }
866. let keepTries = {}
867. for (k of st.tries.keys()) { if (!keys.includes(k)) { keepTries[k] = st.tries[k] } }
868. let kept = []
869. for (e of rstore.entries) {
870. let srcKey = e.source == null || e.source.kind != 'ticket' ? '' : e.source.event
871. if (keys.includes(srcKey)) { console.log('librarian: --redo removes ' + e.id + ' ("' + clip(e.quotes.join(' · '), 80) + '")') } else { kept.push(e) }
872. }
873. rstore.entries = kept
874. saveStore(c.dir, rstore)
875. st.done = keepDone
876. st.tries = keepTries
877. st.cursorMs = 0
878. console.log('librarian: --redo ' + keys.length + ' text(s): ' + keys.join(', '))
879. }
880. if (argv.includes('--retry-refused')) {
881. let keep = {}
882. for (k of st.done.keys()) { if (!st.done[k].startsWith('refused')) { keep[k] = st.done[k] } }
883. st.done = keep
884. st.tries = {}
885. st.cursorMs = 0
886. }
887. let names = creatorNames()
888. let g = gatherTexts(names, floor, sinceText != null ? 0 : st.cursorMs, st.done)
889. if (g.error != null) {
890. console.log('librarian: ERROR — ' + g.error + ' (nothing written, the next pass tries again)')
891. done({ error = g.error })
892. return false
893. }
894. let texts = g.texts
895. if (!o.auto || texts.length > 0) { console.log('librarian: ' + texts.length + ' new creator text(s) (creator = ' + names.join(' / ') + ') · read ' + g.read + ' of ' + g.total + ' ticket(s) updated since ' + (floor > st.cursorMs && sinceText != null ? timestamp(floor).slice(0, 19) + 'Z (--since)' : (st.cursorMs == 0 ? 'the beginning' : timestamp(st.cursorMs).slice(0, 19) + 'Z')) + ' · ' + baseUrl()) }
896. if (!o.post) {
897. for (x of texts) { console.log('LIBRARIAN TEXT ' + x.ref + ' · ' + x.what + ' · ' + x.when + ' · "' + clip(x.quotes.join(' · '), 100) + '"') }
898. console.log('LIBRARIAN DRY RUN — ' + texts.length + ' new creator text(s); no model call, nothing written (--post-files classifies them and writes the files)')
899. done({ newTexts = texts.length })
900. return true
901. }
902. if (texts.length == 0) {
903. st.cursorMs = g.cursorMs
904. st.floorMs = floor
905. st.passes = st.passes + 1
906. st.lastPass = timestamp(now())
907. saveState(c.dir, st)
908. let w = []
909. if (!o.auto) {
910. let store0 = loadStore(c.dir)
911. if (store0.error == null) { w = renderAll(store0, reg) }
912. for (l of w) { console.log('LIBRARIAN FILE ' + l) }
913. console.log('LIBRARIAN: nothing new — no model call · ' + w.length + ' file(s) written, the others unchanged')
914. }
915. done({ newTexts = 0 written = w cost = 0 left = 0 })
916. return true
917. }
918. let lk = takeLock(c.dir)
919. if (lk.error != null) {
920. console.log('librarian: BUSY — ' + lk.error)
921. done({ error = lk.error })
922. return false
923. }
924. let store = loadStore(c.dir)
925. if (store.error != null) {
926. dropLock(c.dir)
927. console.log('librarian: ' + store.error)
928. done({ error = store.error })
929. return false
930. }
931. let sum = { newTexts = texts.length decisions = 0 not = 0 refused = 0 failed = 0 cost = 0 calls = 0 written = [] superseded = 0 left = texts.length }
932. let changed = false
933. let idx = 0
934. let limit = c.max > 0 && c.max < texts.length ? c.max : texts.length
936. let finish = (why) => {
937. // the cursor moves only when every text of this pass got an outcome (a capped / stopped pass reads them again)
938. let left = 0
939. for (x of texts) { if (st.done[x.key] == null) { left = left + 1 } }
940. if (left == 0) { st.cursorMs = g.cursorMs }
941. sum.left = left
942. st.floorMs = floor
943. st.passes = st.passes + 1
944. st.lastPass = timestamp(now())
945. saveState(c.dir, st)
946. if (changed || !o.auto) {
947. sum.written = renderAll(store, reg)
948. for (l of sum.written) { console.log('LIBRARIAN FILE ' + l) }
949. }
950. dropLock(c.dir)
951. console.log('LIBRARIAN: ' + sum.calls + ' of ' + texts.length + ' new text(s) classified' + (why == null ? '' : ' (' + why + ')') + ' · ' + sum.decisions + ' decision(s), ' + sum.superseded + ' superseded, ' + sum.not + ' not a decision, ' + sum.refused + ' refused, ' + sum.failed + ' without an answer · ' + sum.written.length + ' file(s) written')
952. console.log('LIBRARIAN COST: ' + money(sum.cost) + ' for ' + sum.calls + ' model call(s) (' + c.model + ')')
953. done(sum)
954. return null
955. }
957. let step = null
958. step = () => {
959. if (idx >= limit) {
960. finish(limit < texts.length ? 'limit ' + limit + ' per pass — the rest next pass' : null)
961. return null
962. }
963. if (o.stop != null && o.stop()) {
964. finish('stop requested — the rest next pass')
965. return null
966. }
967. let x = texts[idx]
968. idx = idx + 1
969. let allowedHere = []
970. for (n of allowed) { allowedHere.push(n) }
971. if (!allowedHere.includes(x.project)) { allowedHere.push(x.project) }
972. allowedHere.push('all')
973. console.log('LIBRARIAN TEXT ' + x.ref + ' · ' + x.what + ' · ' + x.author + ' · ' + x.when + ' · "' + clip(x.quotes.join(' · '), 100) + '"')
974. classify(c, x, allowedHere, store.entries, (r) => {
975. sum.calls = sum.calls + 1
976. if (r.cost != null) { sum.cost = sum.cost + r.cost }
977. // mission 034: the usage limit — no try counted, the text stays unfiled, the pass stops here (no further call)
978. if (r.limited == true) {
979. sum.limited = true
980. sum.resetMs = r.resetMs
981. sum.limitText = r.limitText
982. console.log(' → USAGE LIMIT — ' + r.limitText + ' · resets ' + (r.resetMs == null ? 'at an unknown time' : timestamp(r.resetMs).slice(0, 19) + 'Z') + ' → the pass stops, the text stays unfiled (no try counted)')
983. finish('Claude usage limit reached — the rest after the reset')
984. return null
985. }
986. let tries = st.tries[x.key] == null ? 1 : st.tries[x.key] + 1
987. let tr = st.tries
988. tr[x.key] = tries
989. st.tries = tr
990. let dn = st.done
991. if (r.reason != null) {
992. sum.failed = sum.failed + 1
993. if (tries >= 3) { dn[x.key] = 'failed 3 times: ' + clip(r.reason, 200) }
994. console.log(' → NO ANSWER (' + tries + ' of 3 tries) — ' + r.reason)
995. } else {
996. let a = checkAnswer(r.answer, x, allowedHere, store.entries)
997. if (a.problems != null) {
998. sum.refused = sum.refused + 1
999. if (tries >= 2) { dn[x.key] = 'refused twice: ' + clip(a.problems.join('; '), 200) }
1000. console.log(' → REFUSED (' + tries + ' of 2 tries) — ' + a.problems.join('; '))
1001. } else if (a.decision == false) {
1002. sum.not = sum.not + 1
1003. dn[x.key] = 'not a decision: ' + clip(a.why, 200)
1004. console.log(' → not a decision — ' + a.why)
1005. } else {
1006. let eid = 'd-' + x.key
1007. let src = { kind = 'ticket' ref = x.ref url = x.url subject = x.subject what = x.what author = x.author event = x.key }
1008. let entry = { id = eid project = a.project topic = a.topic quotes = a.quotes context = '' means = a.means readingBy = 'librarian' date = x.date ms = x.createdMs source = src added = now() status = 'current' why = a.why }
1009. // corrections: the entries it supersedes move to the archive (kept, marked)
1010. let es = []
1011. for (e of store.entries) {
1012. if (a.supersedes.includes(e.id)) {
1013. let ee = e
1014. ee.status = 'superseded'
1015. ee.supersededBy = eid
1016. ee.supersededOn = x.date
1017. es.push(ee)
1018. sum.superseded = sum.superseded + 1
1019. console.log(' → supersedes ' + e.id + ' ("' + clip(e.quotes.join(' · '), 80) + '")')
1020. } else { es.push(e) }
1021. }
1022. es.push(entry)
1023. store.entries = es
1024. changed = true
1025. sum.decisions = sum.decisions + 1
1026. dn[x.key] = 'decision ' + eid
1027. console.log(' → DECISION ' + eid + ' · ' + a.project + ' · ' + a.topic + ' · quote "' + clip(a.quotes.join(' · '), 80) + '" · means: ' + a.means)
1028. if (a.quoteNote != '') { console.log(' → note: ' + a.quoteNote) }
1029. if (a.dropped.length > 0) { console.log(' → note: unknown / not current ids in supersedes dropped: ' + a.dropped.join(', ')) }
1030. saveStore(c.dir, store)
1031. }
1032. }
1033. st.done = dn
1034. saveState(c.dir, st)
1035. step()
1036. return null
1037. })
1038. return null
1039. }
1040. step()
1041. return true
1042. }
1044. // ORDER for the commands that build a brief by hand (`brief`, `work`, `cycle` — scheduler.hl): one pass FIRST, awaited,
1045. // so a decision the creator wrote in any ticket a moment ago is in DECISIONS.md before the brief is assembled. Skipped
1046. // for `--dry-run` (no model call there) and with `--librarian off` / COLONY_LIBRARIAN=off. A failed / busy pass is said
1047. // and the command goes on with the files as they are. `then()` runs once.
1048. static librarianFirst = (argv, what, then) => {
1049. let off = setting(argv, '--librarian', 'COLONY_LIBRARIAN', 'on') == 'off'
1050. if (off || argv.includes('--dry-run')) {
1051. then()
1052. return null
1053. }
1054. librarianPass({ argv = argv post = true auto = true stop = null }, (s) => {
1055. if (s != null && s.limited == true) { console.log('librarian: the Claude usage limit is reached (resets ' + (s.resetMs == null ? 'at an unknown time' : timestamp(s.resetMs).slice(0, 19) + 'Z') + ') — ' + s.left + ' creator text(s) not filed; ' + what + ' goes on with the files as they are') }
1056. else if (s != null && s.error != null) { console.log('librarian: ' + what + ' goes on WITHOUT a finished librarian pass (' + s.error + ') — the decisions files are as they were') }
1057. else if (s != null && s.left != null && s.left > 0) { console.log('librarian: ' + s.left + ' creator text(s) not filed yet — ' + what + ' goes on with the files as they are') }
1058. then()
1059. return null
1060. })
1061. return null
1062. }
1064. // `librarian …` by hand
1065. static runLibrarian = (argv) => {
1066. let post = argv.includes('--post-files')
1067. librarianPass({ argv = argv post = post auto = false stop = null }, (s) => { return null })
1068. return true
1069. }