run.sh

3041 lines

1#!/bin/bash

3# run.sh - Main orchestrator for neocities-modernization pipeline

5# Runs the complete poem processing pipeline from input files to generated HTML.

6# Supports selective stage execution via CLI flags, with stages running in

7# pipeline order regardless of argument order.

9# The full pipeline has 10 stages:

10# 1. Update Words - Sync input files from words repository

11# 2. Extract - Extract content from backup archives

12# 3. Parse - Generate poems.json from sources

13# 4. Validate - Validate poem data

14# 5. Catalog Images - Generate image-catalog.json

15# Durations are deliberately absent from this list. Every stage's wall-clock is

16# measured and recorded to .stage-timings (Issue 10-051), and --dry-run or

17# --verbose prints the average of the last five runs per stage. Hardcoded

18# estimates here had drifted by one to two orders of magnitude -- "~42 hours" for

19# a stage measuring ~40 minutes, "~30 min" for one measuring ~17 seconds -- and

20# stage 8 carried two contradictory numbers in this same file. A measured number

21# cannot rot; an estimate in a comment always does.

22# 6. Embeddings - Generate poem embeddings via the inference server

23# 7. Similarity - Build similarity matrix

24# 8. Diversity - Pre-compute diversity cache

25# 9. Generate HTML - Generate poem pages, gallery, and source browser

26# 10. Generate WordCloud - Generate the word-cloud menu and per-word pages

27#

28# Stages are selected individually with named flags (--extract,

29# --generate-diversity, etc.) or by stage number (--stage 8,

30# --stage=5). Use --full to run all 10 stages.

31#

32# Usage: ./run.sh [FLAGS] [PROJECT_DIR]

33#

34# ── The no-defaults contract (Issue 10-065) ─────────────────────────────────

35# Every value this script hands to a stage must arrive on the command line.

36# There are no invented numbers here, and nothing is read from config.lua to

37# stand in for a flag the operator did not type. The reason is reproducibility:

38# a guessed value is invisible in the output, so two runs typed identically

39# could build different websites with nothing in the log to explain it.

40#

41# When values are absent the script does NOT stop at the first one. It works out

42# every value the SELECTED stages will consume, collects all the absent ones, and

43# prints them together -- so one run tells the operator the whole command they

44# should have typed. See the "Required values" fold below.

45#

46# On/off flags (--force, --verbose, --quiet, --dry-run, --debug, --low-priority)

47# are exempt, and the distinction is not arbitrary: for those, absence IS the

48# value. Nothing is consulted to decide "off". Compare --boosts, whose absence

49# used to mean "go ask config.privacy.include_boosts" -- that is a fallback, so

50# it became a required yes/no.

51#

52# Two flags keep a DERIVED default, and the difference is worth naming: --dir and

53# --output are computed once, from the project root, into a single variable that

54# every path below is then built from. A default computed in one visible place is

55# a different thing from a fallback consulted at ten scattered call sites.

56#

57# ── NEVER put a double quote inside an inline Lua block ────────────────────

58# Several stages run Lua as an inline chunk, passed to luajit with its -e flag

59# followed by a double-quoted string. (That pattern is spelled out indirectly

60# here on purpose: the checker below greps for it, and a comment containing it

61# verbatim would match itself.) The Lua source sits inside a shell

62# DOUBLE-QUOTED string, so the first stray " ends the

63# argument. Everything after it becomes separate shell words handed to luajit as

64# script arguments -- and modules that infer their project root from arg[1]

65# (src/semantic-color-calculator.lua does) then take one of those words as the

66# root. The observed symptom was a stage dying on "Failed to load config from

67# is/config.lua", where "is" was the third word of an ENGLISH COMMENT inside the

68# chunk. bash -n cannot catch it: the script is still syntactically valid.

69#

70# Use single quotes, or no quotes, in anything inside those blocks -- including

71# comments, which are the easy place to forget. To check every block at once,

72# run scripts/check-inline-lua-quotes; every block must report exactly 2 quotes,

73# the opening and the closing one and nothing else.

75# {{{ setup_dir_path

76# The project-root convention: a hard-coded path that any argument overrides, so

77# the script runs correctly from any working directory. This is NOT one of the

78# value fallbacks Issue 10-065 removed -- it is the location of the project

79# itself, which must be known before the script can read anything at all

80# (including the config that a fallback would have consulted).

81setup_dir_path() {

82 if [ -n "$1" ]; then

83 echo "$1"

84 else

85 echo "/mnt/mtwo/programming/ai-stuff/neocities-modernization"

86 fi

87}

88# }}}

90# {{{ Signal handling

91# Trap Ctrl+C so the script actually exits when the operator interrupts.

92# Bash on its own does not always propagate SIGINT to long-running children

93# (luajit's tight inner loops in particular eat the signal), so we kill

94# every background job in our process group and exit non-zero. Exit code

95# 130 is the conventional value for "terminated by SIGINT" (128 + signal#).

96cleanup_on_interrupt() {

97 echo

98 echo "Interrupted by user (SIGINT)" >&2

99 # Kill anything we backgrounded; suppress errors when there are none.

100 jobs -p | xargs -r kill 2>/dev/null

101 # Best-effort kill the entire process group too, in case a child

102 # spawned its own children without forwarding signals.

103 kill -- -$$ 2>/dev/null

104 exit 130

105}

106trap cleanup_on_interrupt INT TERM

107

108# WE_STARTED_INFERENCE_SERVER tracks whether THIS run started the llama.cpp

109# server itself (because validation failed at startup). If it's true, the

110# EXIT trap below shuts the server down again. If the operator (or a prior

111# run) was already running a server when we started, we leave it alone —

112# never kill what we did not start.

113WE_STARTED_INFERENCE_SERVER=false

114

115# cleanup_inference_server: gracefully terminate the llama.cpp server we

116# auto-started during the pre-flight validation phase. Runs on every exit

117# path (normal completion, SIGINT/SIGTERM via cleanup_on_interrupt, errors

118# that hit `exit`). The PID is read from a file the start script writes;

119# if the file is missing or stale (PID no longer alive) we silently bow

120# out — this is best-effort cleanup, not a contract.

121cleanup_inference_server() {

122 if ! $WE_STARTED_INFERENCE_SERVER; then

123 return

124 fi

125 local pid_file="$DIR/tmp/shared-memory/llamacpp-server.pid"

126 if [ ! -f "$pid_file" ]; then

127 return

128 fi

129 local pid

130 pid=$(cat "$pid_file" 2>/dev/null)

131 if [ -z "$pid" ] || ! kill -0 "$pid" 2>/dev/null; then

132 # Stale PID file — clean it up and move on.

133 rm -f "$pid_file"

134 return

135 fi

136 echo "Shutting down inference server (PID $pid) that this run started..." >&2

137 kill "$pid" 2>/dev/null

138 # Give the server up to 5 s to exit on SIGTERM. Most well-behaved

139 # processes shut down within a second; the timeout is generous.

140 local i=0

141 while [ "$i" -lt 5 ]; do

142 if ! kill -0 "$pid" 2>/dev/null; then

143 break

144 fi

145 sleep 1

146 i=$((i + 1))

147 done

148 if kill -0 "$pid" 2>/dev/null; then

149 echo " server did not exit on SIGTERM; sending SIGKILL" >&2

150 kill -9 "$pid" 2>/dev/null

151 fi

152 rm -f "$pid_file"

153}

154

155# {{{ Build-record rollback (Issue 10-065)

156# The build record (output/generation-metadata.json) is written BEFORE the stages

157# run, so that a build interrupted halfway still leaves its parameters behind.

158# The cost of writing early is that a run which fails immediately replaces a

159# record describing a real build with one describing nothing -- demonstrated

160# during this issue's development, when a mistyped --model was caught by the

161# stage-9 embeddings check AFTER the record had already been overwritten.

162#

163# The resolution keeps the early write and makes it reversible: the record about

164# to be replaced is copied into the RAM tier first, and put back if the run does

165# not reach the end. Both properties at once -- an interrupted build leaves its

166# parameters, and a failed build leaves the previous record intact.

167#

168# These are empty until write_generation_metadata caches something, so the

169# restore is a no-op on every run that writes no record (stages 1-4), on --help,

170# and on a failure before the record is reached.

171METADATA_TARGET=""

172METADATA_BACKUP=""

173

174restore_previous_generation_metadata() {

175 local status="$1"

176 # Reaching the end means the new record is the true one; keep it.

177 [ "$status" -eq 0 ] && return

178 # Nothing was cached: either no record was written this run, or none existed

179 # to begin with. In both cases there is nothing to put back.

180 [ -z "$METADATA_BACKUP" ] && return

181 [ -f "$METADATA_BACKUP" ] || return

182 if cp "$METADATA_BACKUP" "$METADATA_TARGET"; then

183 echo "Build did not complete; restored the previous build record." >&2

184 echo " ($METADATA_TARGET)" >&2

185 else

186 echo "Warning: could not restore the previous build record from" >&2

187 echo " $METADATA_BACKUP -- it is still there, in RAM, until reboot." >&2

188 fi

189}

190# }}}

191

192# One EXIT trap, because bash has only one: setting a second would silently

193# replace the first. Capturing $? must be the very first thing this does --

194# any command run before it overwrites the status being reported on.

195on_exit() {

196 local status=$?

197 restore_previous_generation_metadata "$status"

198 cleanup_inference_server

199}

200trap on_exit EXIT

201# }}}

202

203# {{{ TUI Library

204# The interactive menu library, which -I needs. Sourced here (before anything

205# runs) because a shell function must be defined before it is called.

206#

207# Issue 10-065: the load is conditional but the CONSEQUENCE of a failed load is

208# not. Previously a missing library set TUI_AVAILABLE=false and -I quietly ran a

209# different, much older interactive mode instead -- the operator asked for one

210# program and silently got another. Now the load stays conditional (a machine

211# without the library must still be able to run every non-interactive stage) but

212# interactive_mode_tui hard-errors if the library is not actually here. The check

213# lives at the point of use, which is the only place that knows it was needed.

214LIBS_DIR="/home/ritz/programming/ai-stuff/scripts/libs"

215if [[ -f "${LIBS_DIR}/lua-menu.sh" ]] && command -v luajit &>/dev/null; then

216 source "${LIBS_DIR}/lua-menu.sh"

217fi

218# }}}

219

220# {{{ show_help

221# {{{ show_help

222# Reference, not rationale. Every flag gets one line saying what it does and

223# which stages need it; WHY a flag works the way it does belongs in the comments

224# at its implementation, not in front of someone who just wants the spelling.

225show_help() {

226 cat << 'EOF'

227Usage: ./run.sh [FLAGS] [PROJECT_DIR]

228

229Runs the poem processing pipeline. Pick stages by name or number; they always

230run in pipeline order. Values are required per-stage -- run with just your

231stage flags and it will list what else it needs.

232

233Pipeline Stages:

234 --update-words Stage 1: Sync input files from words repository

235 --extract Stage 2: Extract content from backup archives

236 --parse Stage 3: Parse poems into poems.json

237 --validate Stage 4: Run poem validation

238 --catalog-images Stage 5: Catalog images from input directories

239 --generate-embeddings Stage 6: Generate embeddings via the inference server

240 --generate-similarity Stage 7: Build similarity matrix

241 --generate-diversity Stage 8: Pre-compute diversity cache

242 --generate-html Stage 9: Generate poem pages, gallery, source browser

243 --generate-wordcloud Stage 10: Generate word-cloud menu and per-word pages

244

245 --stage N Select a stage by number (--stage 8, --stage=5)

246 --full Run all stages 1-10

247

248Required values ([n] = the stages that need it):

249 --threads N Worker thread count [7,9]

250 --pages N Pages generated per poem [7,8,9]

251 --poems-per-page N Poems per similar/different page [7,8,9]

252 --chrono-per-page N Poems per chronological page [9,10]

253 --wordcloud-words N Words in the cloud, or "all" [6,10]

254 --wordcloud-poems N Poems per word-cloud page [10]

255 --model NAME Embedding model [6,7,8,9,10]

256 --server NAME Inference server, by name from config.lua [6]

257 --boosts yes|no Include fediverse boosts/reblogs [2,3]

258

259Optional:

260 --seed N Master randomization seed. Randomized and recorded

261 if omitted.

262 --force Force regeneration even if files are fresh

263 --force-stage N Force regenerate one stage only (1-10)

264 --dir PATH Assets directory (default: <project>/assets)

265 --output PATH Site output directory (default: <project>/output)

266

267Output Control:

268 --quiet Suppress progress messages

269 --verbose Show detailed progress

270 --dry-run Show what would run, without running it

271 --debug Write logs to output/debug-logs/ and keep them

272 --low-priority Run heavy stages at nice -n 10

273

274Other:

275 -I, --interactive Launch the TUI (with command preview)

276 --list-servers List inference servers and exit

277 --list-external List configured external file sources

278 --sync-only NAME Sync one external source and exit

279 -h, --help Show this message

280

281Examples:

282 ./run.sh --generate-html # lists the values it still needs

283 ./run.sh --stage 4 # validation needs no values

284 ./run.sh --full --threads 8 --pages 3 --poems-per-page 33 \

285 --chrono-per-page 7 --wordcloud-words all --wordcloud-poems 50 \

286 --model nomic-embed-text-v1.5 --server local --boosts no

287

288Notes:

289 - Stage timings: run with --dry-run or --verbose; the plan shows each

290 stage's measured average from .stage-timings.

291 - Caches are written under assets/embeddings/<model>/.

292 - The generated site is large; check `du -sh output/` before deploying.

293EOF

294}

295# }}}

296# }}}

297

298# {{{ Parse command line arguments

299DIR=""

300ASSETS_DIR=""

301OUTPUT_DIR=""

302INTERACTIVE=false

303

304# Stage flags (boolean)

305UPDATE_WORDS=false

306EXTRACT=false

307PARSE=false

308VALIDATE=false

309CATALOG_IMAGES=false

310GENERATE_EMBEDDINGS=false

311GENERATE_SIMILARITY=false

312GENERATE_DIVERSITY=false

313GENERATE_HTML=false

314GENERATE_WORDCLOUD=false

315

316# Config flags. Every one of these starts EMPTY and stays empty unless the

317# operator types it (Issue 10-065). Empty is not "use a default" -- empty is the

318# state the requirement gate below reports as missing. Nothing in this script

319# substitutes a value for an empty one.

320THREADS=""

321# Boost inclusion: the string "yes" or "no", from --boosts. There used to be two

322# mechanisms for this -- an INCLUDE_BOOSTS boolean read by extraction and a

323# BOOSTS_ARG string forwarded to parsing -- fed by three flag spellings, one of

324# which appeared TWICE in the case statement below. Bash case takes the first

325# match, so the second branch was unreachable and --include-boosts only ever

326# reached the parse stage, never extraction. One flag, one variable, both stages.

327BOOSTS=""

328FORCE=false

329# Issue 10-016: Per-stage force flags

330FORCE_STAGE_1=false

331FORCE_STAGE_2=false

332FORCE_STAGE_3=false

333FORCE_STAGE_4=false

334FORCE_STAGE_5=false

335FORCE_STAGE_6=false

336FORCE_STAGE_7=false

337FORCE_STAGE_8=false

338FORCE_STAGE_9=false

339FORCE_STAGE_10=false

340QUIET=false

341VERBOSE=false

342DRY_RUN=false

343# --debug: route all logs to output/ (durable disk) instead of the RAM-backed

344# tmp/ symlink, and preserve them on exit. Added to diagnose a hard GPU lock:

345# such a freeze forces a power-cycle, and the tmpfs-backed tmp/ is wiped on

346# reboot, taking every diagnostic with it. See the setup block after `cd $DIR`.

347DEBUG=false

348# Issue 10-028: Lower process priority for UI responsiveness

349LOW_PRIORITY=false

350# The embedding model. Issue 10-065 made --model required, which collapsed what

351# used to be two variables into one: there was a CLI_MODEL ("what the operator

352# typed, if anything") and a MODEL_NAME ("what we resolved, possibly from

353# config.lua"), and the gap between them was exactly where a --model could go

354# missing. With the flag required they are the same value by construction, so

355# only MODEL_NAME survives. It is still recorded on the per-run notepad

356# (tmp/shared-memory/run-overrides.lua) because child programs resolve the model

357# through that notepad rather than through argv.

358MODEL_NAME=""

359# Issue 8-022: Pagination settings for HTML generation

360PAGES=""

361POEMS_PER_PAGE=""

362# Poems per chronological page. Declared here rather than springing into

363# existence inside the case statement, so the requirement gate can see it.

364CHRONO_PER_PAGE=""

365

366# Issue 10-058: master seed for all randomization (word-cloud shuffle, image

367# order). Issue 10-065 removed the two lower tiers of its old resolution chain.

368# It used to be: --seed, else config.randomization.seed, else a number mixed from

369# the clock and the process id. That third tier is the reason the change was

370# worth making -- it MANUFACTURED a value that had never existed before, so the

371# operator who typed the command did not choose the seed and could not have

372# predicted it. The build was recorded afterward, which makes it reproducible in

373# hindsight but not intentional. Now --seed is required and there is one tier.

374RANDOM_SEED=""

375

376# Issue 8-043: Word cloud configuration

377# Word-cloud word count: a number, or the literal "all" for every word. Both the

378# CLI (--wordcloud-words all) and the menu's "All Words" checkbox set this single

379# value -- there is no separate "all" flag to keep in sync.

380WORDCLOUD_WORDS=""

381# Issue 8-050d: Poems per word-cloud page

382WORDCLOUD_POEMS=""

383

384# Issue 10-003b: External file management

385LIST_EXTERNAL=false

386SYNC_ONLY=""

387

388# Issue 10-017: Inference server configuration

389INFERENCE_SERVER=""

390LIST_SERVERS=false

391

392# Track if any stage flag was explicitly set

393STAGE_FLAG_SET=false

394

395# {{{ take_flag_value()

396# Validate the value half of a "--flag value" pair and leave it in FLAG_VALUE.

397#

398# Issue 10-065: two shapes of mistake used to pass silently, and the second is

399# the dangerous one.

400# ./run.sh --threads -- value is "" (end of the command line).

401# Harmless-looking: it reads as "absent",

402# and the requirement gate catches it.

403# ./run.sh --threads --pages 5 -- value is "--pages". This one LOOKS

404# supplied. The gate would accept it, the

405# thread count would be the string

406# "--pages", --pages itself would then be

407# absent, and the failure would surface much

408# later inside a child program with an error

409# that names neither flag.

410#

411# Why this writes to a global instead of printing its answer: a command

412# substitution -- THREADS=$(flag_value ...) -- runs the function in a SUBSHELL,

413# where `exit 1` kills only that subshell. The script would carry on with an

414# empty value and no error. Writing to FLAG_VALUE keeps the function in this

415# shell, so its exit is the script's exit.

416FLAG_VALUE=""

417take_flag_value() {

418 local flag="$1"

419 local value="$2"

420 if [ -z "$value" ]; then

421 echo "Error: $flag needs a value and none followed it." >&2

422 echo " Write it as: $flag <value>" >&2

423 exit 1

424 fi

425 if [ "${value:0:2}" = "--" ]; then

426 echo "Error: $flag needs a value, but the next thing on the command" >&2

427 echo " line was '$value', which is another flag." >&2

428 echo " Write it as: $flag <value> $value ..." >&2

429 exit 1

430 fi

431 FLAG_VALUE="$value"

432}

433# }}}

434

435while [[ $# -gt 0 ]]; do

436 case $1 in

437 -h|--help)

438 show_help

439 exit 0

440 ;;

441 -I|--interactive)

442 INTERACTIVE=true

443 shift

444 ;;

445 --dir)

446 take_flag_value "--dir" "$2"; ASSETS_DIR="$FLAG_VALUE"

447 shift 2

448 ;;

449 --dir=*)

450 take_flag_value "--dir" "${1#*=}"; ASSETS_DIR="$FLAG_VALUE"

451 shift

452 ;;

453 --output)

454 take_flag_value "--output" "$2"; OUTPUT_DIR="$FLAG_VALUE"

455 shift 2

456 ;;

457 --output=*)

458 take_flag_value "--output" "${1#*=}"; OUTPUT_DIR="$FLAG_VALUE"

459 shift

460 ;;

461 --threads)

462 take_flag_value "--threads" "$2"; THREADS="$FLAG_VALUE"

463 shift 2

464 ;;

465 --threads=*)

466 take_flag_value "--threads" "${1#*=}"; THREADS="$FLAG_VALUE"

467 shift

468 ;;

469 # Issue 10-058/10-065: the master seed for every randomization site in

470 # this build. The only source now -- config.randomization.seed is no

471 # longer consulted and no seed is ever manufactured.

472 --seed)

473 take_flag_value "--seed" "$2"; RANDOM_SEED="$FLAG_VALUE"

474 shift 2

475 ;;

476 --seed=*)

477 take_flag_value "--seed" "${1#*=}"; RANDOM_SEED="$FLAG_VALUE"

478 shift

479 ;;

480 --force)

481 FORCE=true

482 shift

483 ;;

484 # Issue 10-016: Per-stage force regeneration (space-separated format)

485 --force-stage)

486 stage_num="$2"

487 case "$stage_num" in

488 1) FORCE_STAGE_1=true ;;

489 2) FORCE_STAGE_2=true ;;

490 3) FORCE_STAGE_3=true ;;

491 4) FORCE_STAGE_4=true ;;

492 5) FORCE_STAGE_5=true ;;

493 6) FORCE_STAGE_6=true ;;

494 7) FORCE_STAGE_7=true ;;

495 8) FORCE_STAGE_8=true ;;

496 9) FORCE_STAGE_9=true ;;

497 10) FORCE_STAGE_10=true ;;

498 *)

499 echo "ERROR: Invalid stage number: $stage_num (valid: 1-10)" >&2

500 exit 1

501 ;;

502 esac

503 shift 2

504 ;;

505 # Issue 10-016: Per-stage force regeneration (= format for backward compatibility)

506 --force-stage=*)

507 stage_num="${1#*=}"

508 case "$stage_num" in

509 1) FORCE_STAGE_1=true ;;

510 2) FORCE_STAGE_2=true ;;

511 3) FORCE_STAGE_3=true ;;

512 4) FORCE_STAGE_4=true ;;

513 5) FORCE_STAGE_5=true ;;

514 6) FORCE_STAGE_6=true ;;

515 7) FORCE_STAGE_7=true ;;

516 8) FORCE_STAGE_8=true ;;

517 9) FORCE_STAGE_9=true ;;

518 10) FORCE_STAGE_10=true ;;

519 *)

520 echo "ERROR: Invalid stage number: $stage_num (valid: 1-10)" >&2

521 exit 1

522 ;;

523 esac

524 shift

525 ;;

526 --quiet)

527 QUIET=true

528 shift

529 ;;

530 --verbose)

531 VERBOSE=true

532 shift

533 ;;

534 --dry-run)

535 DRY_RUN=true

536 shift

537 ;;

538 # Boost inclusion (reshared posts), read by both the extraction and the

539 # parse stages -- they change what poems.json contains, so they only take

540 # effect on a (re)parse.

541 #

542 # Issue 10-065 replaced three flag spellings (--include-boosts,

543 # --no-boosts, --exclude-boosts) with one that carries its answer, for

544 # two reasons. First, absence used to mean "read

545 # config.privacy.include_boosts", which is a fallback. Second, the old

546 # --include-boosts was written into this case statement TWICE; bash takes

547 # the first match, so the second branch never ran and the flag reached

548 # only half the pipeline. A single branch cannot be shadowed by itself.

549 --boosts)

550 take_flag_value "--boosts" "$2"; BOOSTS="$FLAG_VALUE"

551 shift 2

552 ;;

553 --boosts=*)

554 take_flag_value "--boosts" "${1#*=}"; BOOSTS="$FLAG_VALUE"

555 shift

556 ;;

557 # --debug: persist logs to output/ (survives the reboot a hard GPU

558 # lock forces). Handled after DIR is resolved, below.

559 --debug)

560 DEBUG=true

561 shift

562 ;;

563 # Issue 10-028: Lower process priority for UI responsiveness

564 --low-priority)

565 LOW_PRIORITY=true

566 shift

567 ;;

568 --model)

569 take_flag_value "--model" "$2"; MODEL_NAME="$FLAG_VALUE"

570 shift 2

571 ;;

572 --model=*)

573 take_flag_value "--model" "${1#*=}"; MODEL_NAME="$FLAG_VALUE"

574 shift

575 ;;

576 # Issue 8-022: Pagination flags for HTML generation

577 --pages)

578 take_flag_value "--pages" "$2"; PAGES="$FLAG_VALUE"

579 shift 2

580 ;;

581 --pages=*)

582 take_flag_value "--pages" "${1#*=}"; PAGES="$FLAG_VALUE"

583 shift

584 ;;

585 --poems-per-page)

586 take_flag_value "--poems-per-page" "$2"; POEMS_PER_PAGE="$FLAG_VALUE"

587 shift 2

588 ;;

589 --poems-per-page=*)

590 take_flag_value "--poems-per-page" "${1#*=}"; POEMS_PER_PAGE="$FLAG_VALUE"

591 shift

592 ;;

593 --chrono-per-page)

594 take_flag_value "--chrono-per-page" "$2"; CHRONO_PER_PAGE="$FLAG_VALUE"

595 shift 2

596 ;;

597 --chrono-per-page=*)

598 take_flag_value "--chrono-per-page" "${1#*=}"; CHRONO_PER_PAGE="$FLAG_VALUE"

599 shift

600 ;;

601 # Issue 8-043: Word cloud configuration. Word count is set with

602 # --wordcloud-words N, or "--wordcloud-words all" for every word.

603 --wordcloud-words)

604 take_flag_value "--wordcloud-words" "$2"; WORDCLOUD_WORDS="$FLAG_VALUE"

605 shift 2

606 ;;

607 --wordcloud-words=*)

608 take_flag_value "--wordcloud-words" "${1#*=}"; WORDCLOUD_WORDS="$FLAG_VALUE"

609 shift

610 ;;

611 # Issue 8-050d: Poems per word-cloud page

612 --wordcloud-poems)

613 take_flag_value "--wordcloud-poems" "$2"; WORDCLOUD_POEMS="$FLAG_VALUE"

614 shift 2

615 ;;

616 --wordcloud-poems=*)

617 take_flag_value "--wordcloud-poems" "${1#*=}"; WORDCLOUD_POEMS="$FLAG_VALUE"

618 shift

619 ;;

620 # Issue 8-011/10-065: the second --include-boosts branch was here. It was

621 # unreachable (the first one, above, matched every time) and it fed a

622 # separate INCLUDE_BOOSTS variable that only the extraction stage read --

623 # which is why passing the flag changed parsing but not extraction. Both

624 # branches and both variables are gone; --boosts yes|no replaces them.

625 # Issue 10-003b: External file management

626 --list-external)

627 LIST_EXTERNAL=true

628 shift

629 ;;

630 --sync-only)

631 take_flag_value "--sync-only" "$2"; SYNC_ONLY="$FLAG_VALUE"

632 shift 2

633 ;;

634 --sync-only=*)

635 take_flag_value "--sync-only" "${1#*=}"; SYNC_ONLY="$FLAG_VALUE"

636 shift

637 ;;

638 # Issue 10-017: Inference server configuration

639 --server)

640 take_flag_value "--server" "$2"; INFERENCE_SERVER="$FLAG_VALUE"

641 shift 2

642 ;;

643 --server=*)

644 take_flag_value "--server" "${1#*=}"; INFERENCE_SERVER="$FLAG_VALUE"

645 shift

646 ;;

647 --list-servers)

648 LIST_SERVERS=true

649 shift

650 ;;

651 # Stage flags

652 --update-words)

653 UPDATE_WORDS=true

654 STAGE_FLAG_SET=true

655 shift

656 ;;

657 --extract)

658 EXTRACT=true

659 STAGE_FLAG_SET=true

660 shift

661 ;;

662 --parse)

663 PARSE=true

664 STAGE_FLAG_SET=true

665 shift

666 ;;

667 --validate)

668 VALIDATE=true

669 STAGE_FLAG_SET=true

670 shift

671 ;;

672 --catalog-images)

673 CATALOG_IMAGES=true

674 STAGE_FLAG_SET=true

675 shift

676 ;;

677 --generate-embeddings)

678 GENERATE_EMBEDDINGS=true

679 STAGE_FLAG_SET=true

680 shift

681 ;;

682 --generate-similarity)

683 GENERATE_SIMILARITY=true

684 STAGE_FLAG_SET=true

685 shift

686 ;;

687 --generate-diversity)

688 GENERATE_DIVERSITY=true

689 STAGE_FLAG_SET=true

690 shift

691 ;;

692 --generate-html)

693 GENERATE_HTML=true

694 STAGE_FLAG_SET=true

695 shift

696 ;;

697 --generate-wordcloud)

698 GENERATE_WORDCLOUD=true

699 STAGE_FLAG_SET=true

700 shift

701 ;;

702 # --stage N or --stage=N — select a specific stage by number.

703 # Stage map (numeric): 1=update-words, 2=extract, 3=parse,

704 # 4=validate, 5=catalog-images, 6=generate-embeddings,

705 # 7=generate-similarity, 8=generate-diversity, 9=generate-html,

706 # 10=generate-wordcloud. Can be repeated (e.g. --stage 6 --stage 7).

707 --stage)

708 case "$2" in

709 1) UPDATE_WORDS=true ;;

710 2) EXTRACT=true ;;

711 3) PARSE=true ;;

712 4) VALIDATE=true ;;

713 5) CATALOG_IMAGES=true ;;

714 6) GENERATE_EMBEDDINGS=true ;;

715 7) GENERATE_SIMILARITY=true ;;

716 8) GENERATE_DIVERSITY=true ;;

717 9) GENERATE_HTML=true ;;

718 10) GENERATE_WORDCLOUD=true ;;

719 *) echo "Error: --stage expects a number 1-10, got: $2" >&2; exit 1 ;;

720 esac

721 STAGE_FLAG_SET=true

722 shift 2

723 ;;

724 --stage=*)

725 STAGE_NUM="${1#*=}"

726 case "$STAGE_NUM" in

727 1) UPDATE_WORDS=true ;;

728 2) EXTRACT=true ;;

729 3) PARSE=true ;;

730 4) VALIDATE=true ;;

731 5) CATALOG_IMAGES=true ;;

732 6) GENERATE_EMBEDDINGS=true ;;

733 7) GENERATE_SIMILARITY=true ;;

734 8) GENERATE_DIVERSITY=true ;;

735 9) GENERATE_HTML=true ;;

736 10) GENERATE_WORDCLOUD=true ;;

737 *) echo "Error: --stage expects a number 1-10, got: $STAGE_NUM" >&2; exit 1 ;;

738 esac

739 STAGE_FLAG_SET=true

740 shift

741 ;;

742 --full)

743 # ALL stages including expensive embedding generation (1-10)

744 UPDATE_WORDS=true

745 EXTRACT=true

746 PARSE=true

747 VALIDATE=true

748 CATALOG_IMAGES=true

749 GENERATE_EMBEDDINGS=true

750 GENERATE_SIMILARITY=true

751 GENERATE_DIVERSITY=true

752 GENERATE_HTML=true

753 GENERATE_WORDCLOUD=true

754 STAGE_FLAG_SET=true

755 shift

756 ;;

757 -*)

758 echo "Unknown option: $1" >&2

759 echo "Use --help for usage information" >&2

760 exit 1

761 ;;

762 *)

763 DIR="$1"

764 shift

765 ;;

766 esac

767done

768

769# No implicit stages — require explicit selection. The operator should

770# say what they want to run: a named stage flag, --stage N, or --full.

771if ! $STAGE_FLAG_SET && ! $INTERACTIVE && ! $LIST_SERVERS; then

772 echo "Error: no stages selected. Use --full, a named stage flag" >&2

773 echo " (e.g. --generate-diversity), --stage N, or -I for interactive mode." >&2

774 echo " Run with --help for the full flag list." >&2

775 exit 1

776fi

777# }}}

778

779# {{{ Required values (Issue 10-065)

780# The gate that replaced every default and config fallback in this script.

781#

782# Two properties this is built around, both deliberate:

783#

784# Requirements follow the SELECTED stages. --validate alone requires nothing;

785# --stage 10 requires the word-cloud values and the seed but not the thread

786# count. A gate that demanded every value on every run would be demanding

787# values the run will never read, which teaches the operator to type noise --

788# and noise types just as easily when it is wrong.

789#

790# Absent values are COLLECTED, never fatal on sight. Exiting at the first

791# absence makes the operator play twenty questions: four missing values cost

792# four runs to discover. Every check below records and continues; one report

793# at the end lists the whole set.

794#

795# The table is the honest documentation of what each stage actually reads, which

796# is why it is a table and not a chain of ifs -- a new stage declares its needs

797# on one line, in one place, next to every other stage's.

798#

799# Row format: VARIABLE ; flag usage ; why it is needed ; consumers ; record key

800# where "consumers" is a comma-separated list of STAGE_BOOLEAN:stage-number, and

801# "record key" is the name this value is written under in the build's

802# generation-metadata.json.

803#

804# The record key lives HERE, in the same row as everything else, so the gate and

805# the build record cannot drift apart. Adding a flag means adding one row, and it

806# is then simultaneously required, explained in the missing-values report, and

807# recorded in the artifact -- rather than three edits in three places, of which

808# somebody eventually does two.

809#

810# The field separator is ";" rather than the more obvious "|" because one flag's

811# usage text has to READ the way the operator must type it -- "--boosts yes|no"

812# contains a literal pipe. Choosing a separator that cannot appear in the data

813# keeps the split a plain one-line read instead of positional index arithmetic.

814REQUIRED_VALUES=(

815 "THREADS;--threads N;parallel worker count;GENERATE_SIMILARITY:7,GENERATE_HTML:9;threads"

816 "PAGES;--pages N;pages generated per poem;GENERATE_SIMILARITY:7,GENERATE_DIVERSITY:8,GENERATE_HTML:9;pages"

817 "POEMS_PER_PAGE;--poems-per-page N;poems per similar/different page;GENERATE_SIMILARITY:7,GENERATE_DIVERSITY:8,GENERATE_HTML:9;poems_per_page"

818 "CHRONO_PER_PAGE;--chrono-per-page N;poems per chronological page;GENERATE_HTML:9,GENERATE_WORDCLOUD:10;chrono_per_page"

819 "WORDCLOUD_WORDS;--wordcloud-words N;words in the cloud, or the word 'all';GENERATE_EMBEDDINGS:6,GENERATE_WORDCLOUD:10;wordcloud_words"

820 "WORDCLOUD_POEMS;--wordcloud-poems N;poems per word-cloud page;GENERATE_WORDCLOUD:10;wordcloud_poems"

821 # --seed is NOT in this table: an absent seed is randomized, not refused.

822 # It is still RECORDED (see resolve_random_seed), which is what makes an

823 # unseeded build reproducible after the fact -- the property Issue 10-058

824 # built the seed for. A required seed would mean no build can start without

825 # inventing a number to type, and a number typed to satisfy a prompt is not

826 # a more deliberate choice than one the machine picked and wrote down.

827 "MODEL_NAME;--model NAME;embedding model, and the cache directory it names;GENERATE_EMBEDDINGS:6,GENERATE_SIMILARITY:7,GENERATE_DIVERSITY:8,GENERATE_HTML:9,GENERATE_WORDCLOUD:10;model"

828 "INFERENCE_SERVER;--server NAME;inference server, by name from config.lua;GENERATE_EMBEDDINGS:6;server"

829 "BOOSTS;--boosts yes|no;whether reshared posts are included;EXTRACT:2,PARSE:3;boosts"

830)

831

832# Each entry: "flag usage;reason;stage numbers that wanted it"

833MISSING_VALUES=()

834

835# {{{ collect_missing_values()

836# Walk the table, and for each value work out whether any SELECTED stage

837# consumes it. If one does and the value is empty, record it. Never exits --

838# recording and continuing is the whole point.

839collect_missing_values() {

840 local row var usage reason consumers record_key

841 local entries entry stage_var stage_num wanted_by

842 for row in "${REQUIRED_VALUES[@]}"; do

843 IFS=';' read -r var usage reason consumers record_key <<< "$row"

844

845 # Which of this value's consumer stages did the operator actually select?

846 wanted_by=""

847 IFS=',' read -r -a entries <<< "$consumers"

848 for entry in "${entries[@]}"; do

849 stage_var="${entry%%:*}"

850 stage_num="${entry##*:}"

851 # Indirect expansion: ${!stage_var} reads the variable NAMED by

852 # stage_var, so the table can refer to stage booleans by name.

853 if [ "${!stage_var}" = "true" ]; then

854 wanted_by="${wanted_by:+$wanted_by, }$stage_num"

855 fi

856 done

857

858 # No selected stage reads this value -> not required on this run.

859 [ -z "$wanted_by" ] && continue

860 # Supplied. (Any non-empty string counts here; whether it is a sensible

861 # number is the child program's business, not this gate's.)

862 [ -n "${!var}" ] && continue

863

864 MISSING_VALUES+=("$usage;$reason;$wanted_by")

865 done

866}

867# }}}

868

869# {{{ report_missing_values()

870# Print every absent value at once and stop. The flag spelling starts each line

871# so the whole block can be read straight into a command line; the explanation

872# rides behind a "#" so a copied line is still valid shell.

873report_missing_values() {

874 if [ "${#MISSING_VALUES[@]}" -eq 0 ]; then

875 return 0

876 fi

877

878 # Width pass, so the "#" comments line up and the eye can scan the flags.

879 local row usage reason stages width=0

880 for row in "${MISSING_VALUES[@]}"; do

881 IFS=';' read -r usage reason stages <<< "$row"

882 [ "${#usage}" -gt "$width" ] && width=${#usage}

883 done

884

885 echo "" >&2

886 echo "can't run generation script, missing these flags:" >&2

887 echo "" >&2

888 for row in "${MISSING_VALUES[@]}"; do

889 IFS=';' read -r usage reason stages <<< "$row"

890 # "stage 9" reads wrong for a list; "stages 7, 8, 9" reads wrong for one.

891 local noun="stage"

892 case "$stages" in *,*) noun="stages" ;; esac

893 printf ' %-*s # %s (%s %s)\n' "$width" "$usage" "$reason" "$noun" "$stages" >&2

894 done

895 echo "" >&2

896 echo "Every value the pipeline consumes must be given explicitly." >&2

897 echo "run.sh has no defaults and reads no fallbacks from config.lua." >&2

898 echo "Only the stages you selected are asked about; --help explains which." >&2

899 exit 1

900}

901# }}}

902

903# {{{ validate_supplied_values()

904# Shape checks for values that DID arrive. Separate from the gate above on

905# purpose: "absent" is a list the operator can act on all at once, but "present

906# and malformed" is a typo in a specific place, and pointing at it immediately

907# is more useful than burying it in a list of unrelated absences.

908validate_supplied_values() {

909 # The seed must round-trip through a command line, a JSON file and Lua's

910 # randomseed unchanged, so it is a non-negative integer or it is nothing.

911 # Substituting a working seed for a broken one would defeat the entire

912 # reason the seed exists.

913 if [ -n "$RANDOM_SEED" ]; then

914 case "$RANDOM_SEED" in

915 *[!0-9]*)

916 echo "ERROR: --seed '$RANDOM_SEED' is not a non-negative integer." >&2

917 exit 1

918 ;;

919 esac

920 fi

921

922 # --boosts carries its own answer, so the answer has to be one we know.

923 if [ -n "$BOOSTS" ] && [ "$BOOSTS" != "yes" ] && [ "$BOOSTS" != "no" ]; then

924 echo "ERROR: --boosts takes 'yes' or 'no', not '$BOOSTS'." >&2

925 echo " It decides whether reshared (boosted) posts become poems." >&2

926 exit 1

927 fi

928}

929# }}}

930# }}}

931

932# {{{ Derived on/off settings

933# These translate presence-flags into the shapes other programs want. They are

934# not defaults: an absent --force means off because that is what the flag means,

935# not because anything was consulted to decide it.

936

937# Issue 8-032: Convert FORCE to Lua boolean for passing to Lua functions

938if $FORCE; then

939 FORCE_LUA="true"

940else

941 FORCE_LUA="false"

942fi

943

944# Issue 10-028: Set up nice prefix for low priority execution

945# When enabled, heavy operations run at nice level 10 (lower priority)

946# This keeps the desktop/terminal responsive during long pipeline runs

947NICE_PREFIX=""

948if $LOW_PRIORITY; then

949 NICE_PREFIX="nice -n 10"

950fi

951# }}}

952

953# {{{ Setup directories

954DIR=$(setup_dir_path "$DIR")

955

956# Issue 10-051: stage wall-clock timing. Sourced after DIR is final so the

957# library knows where .stage-timings lives. Provides timed_stage (wrap a stage

958# to record its duration on success) and stage_timing_label (render the measured

959# estimate for the pre-flight list).

960#

961# Issue 10-065: a missing library is now fatal. It used to be replaced by a stub

962# -- `timed_stage() { shift; "$@"; }` -- which ran the stage and recorded

963# nothing. That is the quietest possible failure: the pipeline works, every stage

964# runs, and the only symptom is that the pre-flight time estimates never improve,

965# months later, for reasons nobody can reconstruct.

966if [ ! -f "${DIR}/scripts/stage-timing.sh" ]; then

967 echo "Error: stage-timing library not found: ${DIR}/scripts/stage-timing.sh" >&2

968 echo " It records each stage's wall-clock time to .stage-timings, which" >&2

969 echo " the pre-flight plan reads back as its duration estimates." >&2

970 exit 1

971fi

972source "${DIR}/scripts/stage-timing.sh"

973if ! command -v timed_stage >/dev/null; then

974 echo "Error: ${DIR}/scripts/stage-timing.sh loaded but does not define timed_stage." >&2

975 exit 1

976fi

977

978# {{{ Assets and output directories

979# Issue 10-065: these are the two DERIVED values in the script, and the

980# difference from a fallback is that the derivation happens exactly once, here,

981# and every path below is built from the result.

982#

983# What it replaced: --dir used to be forwarded to main.lua as "--dir PATH" and

984# nowhere else, while this script separately hardcoded "$DIR/assets/poems.json"

985# for its own freshness checks -- so passing --dir pointed the child at one

986# corpus and left the parent checking another. --output was worse: it reached

987# exactly one line (the metadata write) while every stage wrote to "$DIR/output".

988# A flag that is honoured in one place out of ten is not a flag, it is a trap.

989if [ -z "$ASSETS_DIR" ]; then

990 ASSETS_DIR="$DIR/assets"

991fi

992if [ -z "$OUTPUT_DIR" ]; then

993 OUTPUT_DIR="$DIR/output"

994fi

995

996# The child programs take the assets root as "--dir PATH". Passed unconditionally

997# now, because ASSETS_DIR is always resolved -- there is no "omit the flag and

998# let the child decide" case left.

999ASSETS_ARG="--dir $ASSETS_DIR"

1000# }}}

1001

1002# Ensure we're in the right directory

1003cd "$DIR" || {

1004 echo "Error: Could not access directory $DIR" >&2

1005 exit 1

1006}

1007# }}}

1008

1009# {{{ --debug: persistent logging

1010# Why this exists: a hard GPU lock forces a power-cycle, and the tmp/ symlink

1011# points at a tmpfs subdir under /tmp/ (RAM) that the reboot wipes — so the

1012# logs that would explain the freeze are gone before they can be read.

1013# --debug routes logs to output/debug-logs/ (durable disk) instead.

1014#

1015# Two mechanisms, working together:

1016# 1. NEOCITIES_LOG_DIR is exported so the child scripts that own the

1017# inference logs — scripts/start-llamacpp-server.sh (llamacpp-server.log)

1018# and generate-embeddings.sh (embedding_generation.log) — write there

1019# and skip their usual end-of-run log deletion.

1020# 2. This script's own console output is tee'd to run.log, so whatever stage

1021# was mid-flight at the instant of the freeze (including the GPU Vulkan

1022# similarity/diversity stages, which log only to stdout) leaves a trail.

1023#

1024# Caveat worth knowing: on a true hard lock you must hard-power-cycle, and the

1025# kernel may not have flushed the last few seconds of file writes (dirty pages)

1026# to disk. Durable disk still captures vastly more than tmpfs, but the final

1027# line or two before the lock can still be lost.

1028if $DEBUG; then

1029 LOG_DIR="$OUTPUT_DIR/debug-logs"

1030 mkdir -p "$LOG_DIR"

1031 export NEOCITIES_LOG_DIR="$LOG_DIR"

1032 # The Vulkan C library reads VKC_DEBUG to switch its progress bars from the

1033 # animated single-line "\r" form to verbose, newline-terminated lines --

1034 # the right shape when stdout is the fsync-logger pipe below and we want a

1035 # durable, per-line history of a possibly-freezing run.

1036 export VKC_DEBUG=1

1037 # Don't reroute stdout through a pipe in interactive mode: the TUI checks

1038 # isatty() and a pipe would break its rendering. The child-script file

1039 # logs still land in LOG_DIR via the exported env var above.

1040 #

1041 # fsync-logger (not tee) is used so every line is fsync()'d to disk the

1042 # instant it is printed — the stage banners are exactly what triage needs,

1043 # and a hard lock right after a banner must not lose it to a dirty-page

1044 # buffer. Slow, but --debug is for catching a freeze, not for speed.

1045 if ! $INTERACTIVE; then

1046 exec > >("$DIR/scripts/fsync-logger" "$LOG_DIR/run.log") 2>&1

1047 fi

1048 echo "[DEBUG] Logging to $LOG_DIR (per-line fsync to disk; persists across reboots; logs kept on exit)"

1049fi

1050# }}}

1051

1052# {{{ Issue 10-003b: Handle external file commands (immediate actions)

1053if $LIST_EXTERNAL; then

1054 "$DIR/scripts/sync-external-files" --list

1055 exit 0

1056fi

1057

1058if [ -n "$SYNC_ONLY" ]; then

1059 "$DIR/scripts/sync-external-files" "$SYNC_ONLY"

1060 exit $?

1061fi

1062# }}}

1063

1064# {{{ Issue 10-017: Handle Inference server commands (immediate actions)

1065if $LIST_SERVERS; then

1066 luajit -e "

1067 package.path = '$DIR/libs/?.lua;' .. package.path

1068 local inference = require('inference-server-config')

1069 inference.list_servers()

1070 "

1071 exit 0

1072fi

1073# }}}

1074

1075# {{{ Materialize the RAM-backed tmp/ directory

1076# A bare `mkdir -p tmp` does NOT work here: tmp/ is a symlink, and if its target

1077# is missing (wiped on reboot) mkdir sees the link, reports "exists", and creates

1078# nothing. ensure-tmp-symlink is the project's idempotent, fail-loud helper for

1079# exactly this -- it creates the /tmp target the symlink points at. Done early

1080# because the per-run notepad and every stage log live underneath it.

1081"$DIR/scripts/ensure-tmp-symlink" "$DIR" || {

1082 echo "Error: could not materialize the tmp/ RAM directory (scripts/ensure-tmp-symlink)" >&2

1083 exit 1

1084}

1085# }}}

1086

1087# NOTE (Issue 10-065): the embedding-model resolution and the per-run overrides

1088# notepad used to sit here. They moved BELOW the interactive menu and the

1089# requirement gate, and the ordering is the point: the old code resolved the

1090# model by asking config.lua whenever --model was absent, which is precisely the

1091# fallback this issue removed. Resolution cannot come before the check that the

1092# operator supplied the value -- and the check cannot come before the menu, which

1093# is the other way values arrive. See "Record this run's choices" further down.

1094

1095# {{{ Logging functions

1096log_info() {

1097 if ! $QUIET; then

1098 echo "$1"

1099 fi

1100}

1101

1102log_verbose() {

1103 if $VERBOSE; then

1104 echo "$1"

1105 fi

1106}

1107

1108log_stage() {

1109 if ! $QUIET; then

1110 echo ""

1111 echo -e "${COLOR_MAGENTA}═══════════════════════════════════════════════════════════════════${COLOR_RESET}"

1112 echo -e " ${COLOR_GREEN}$1${COLOR_RESET}"

1113 echo -e "${COLOR_MAGENTA}═══════════════════════════════════════════════════════════════════${COLOR_RESET}"

1114 fi

1115}

1116

1117log_dry_run() {

1118 echo "[DRY-RUN] Would execute: $1"

1119}

1120

1121# ANSI color codes for terminal output

1122# These add visual distinction to success/info/error messages

1123COLOR_GREEN="\033[92m" # Bright green for success (✓, ✅)

1124COLOR_BLUE="\033[94m" # Bright blue for info (ℹ️)

1125COLOR_RED="\033[91m" # Bright red for errors (✗, ❌)

1126COLOR_YELLOW="\033[93m" # Bright yellow for warnings (⚠️)

1127COLOR_MAGENTA="\033[95m" # Bright magenta for stage delimiters

1128COLOR_RESET="\033[0m" # Reset to default

1129

1130# Colored symbol helpers

1131symbol_success() {

1132 echo -e "${COLOR_GREEN}$1${COLOR_RESET}"

1133}

1134

1135symbol_info() {

1136 echo -e "${COLOR_BLUE}$1${COLOR_RESET}"

1137}

1138

1139symbol_error() {

1140 echo -e "${COLOR_RED}$1${COLOR_RESET}"

1141}

1142

1143symbol_warning() {

1144 echo -e "${COLOR_YELLOW}$1${COLOR_RESET}"

1145}

1146# }}}

1147

1148# {{{ Issue 10-058: record the build's master seed

1149# A single integer governs every randomization site this run (the word-cloud

1150# shuffle and image-order randomization).

1151#

1152# Issue 10-065 deleted the resolver that used to live here. It tried --seed

1153# first, then config.randomization.seed, then MANUFACTURED one by mixing the

1154# clock with the process id. That last tier is why this mattered more than the

1155# other fallbacks: it invented a value that had never existed, so a build's most

1156# consequential input was chosen by nobody. It was recorded afterward, which

1157# makes such a build reproducible in hindsight but not intentional -- and the

1158# only way to learn what governed your word cloud was to read the metadata file

1159# after the fact. Now --seed is required by the stages that randomize, and the

1160# recording below documents a decision instead of disclosing an accident.

1161

1162# {{{ write_generation_metadata()

1163# The record of what produced this build. A small JSON at the output root;

1164# written early (so an interrupted build still leaves it) and at the root (so

1165# per-stage clears, which only touch output/ subdirs, never wipe it).

1166#

1167# Issue 10-065: it used to record the seed, pages and poems-per-page only. That

1168# gap had a cost that was paid in full: after the word-cloud pages were lost,

1169# nothing in the build's own artifacts said what --wordcloud-words,

1170# --wordcloud-poems or --chrono-per-page had produced them, so 11 GB -- two

1171# thirds of the site -- could not be reproduced from its own record.

1172#

1173# It now writes every value the requirement gate asked for on this run, driven by

1174# the SAME table. That is the property worth keeping: the record cannot fall

1175# behind the flags, because adding a flag to REQUIRED_VALUES adds it here too.

1176#

1177# Only values this run actually needed are written. A run of stages 9 and 10 has

1178# no --server, and recording an empty one would claim something untrue about how

1179# the build was made.

1180write_generation_metadata() {

1181 mkdir -p "$OUTPUT_DIR"

1182

1183 # Cache the record we are about to replace, in the RAM tier, so the EXIT trap

1184 # can put it back if this run does not finish. RAM is the right home: the

1185 # copy is meaningful only for the duration of this run, and a reboot that

1186 # wipes it also ends the run it belonged to.

1187 METADATA_TARGET="$OUTPUT_DIR/generation-metadata.json"

1188 if [ -f "$METADATA_TARGET" ]; then

1189 METADATA_BACKUP="$DIR/tmp/shared-memory/generation-metadata.previous.json"

1190 cp "$METADATA_TARGET" "$METADATA_BACKUP" || {

1191 # Refuse rather than overwrite irreversibly. The existing record is

1192 # the only description of what is currently in output/; replacing it

1193 # with no way back is worse than not recording this run at all.

1194 echo "Error: could not cache the existing build record to" >&2

1195 echo " $METADATA_BACKUP" >&2

1196 echo " Refusing to overwrite $METADATA_TARGET without a way back." >&2

1197 exit 1

1198 }

1199 fi

1200

1201 local generated_at

1202 generated_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)

1203

1204 # Which stages ran, by number, in pipeline order.

1205 local stages=()

1206 $UPDATE_WORDS && stages+=(1)

1207 $EXTRACT && stages+=(2)

1208 $PARSE && stages+=(3)

1209 $VALIDATE && stages+=(4)

1210 $CATALOG_IMAGES && stages+=(5)

1211 $GENERATE_EMBEDDINGS && stages+=(6)

1212 $GENERATE_SIMILARITY && stages+=(7)

1213 $GENERATE_DIVERSITY && stages+=(8)

1214 $GENERATE_HTML && stages+=(9)

1215 $GENERATE_WORDCLOUD && stages+=(10)

1216 local stages_json

1217 stages_json=$(IFS=,; echo "${stages[*]}")

1218

1219 # Every required value that applied, as "key": "value" lines. Values are

1220 # emitted as JSON strings even when numeric: "all" is a legitimate

1221 # --wordcloud-words, so the column is not uniformly a number, and a reader

1222 # that must handle both is better served by one consistent type.

1223 local row var usage reason consumers record_key

1224 local entries entry stage_var wanted value_lines=()

1225 for row in "${REQUIRED_VALUES[@]}"; do

1226 IFS=';' read -r var usage reason consumers record_key <<< "$row"

1227 wanted=false

1228 IFS=',' read -r -a entries <<< "$consumers"

1229 for entry in "${entries[@]}"; do

1230 stage_var="${entry%%:*}"

1231 [ "${!stage_var}" = "true" ] && wanted=true

1232 done

1233 $wanted || continue

1234 [ -z "${!var}" ] && continue

1235 value_lines+=(" \"$record_key\": \"${!var}\"")

1236 done

1237

1238 # Join with ",\n". NOT "${value_lines[*]}" with IFS=$',\n' -- bash uses only

1239 # the FIRST character of IFS as the separator for [*], so that produced one

1240 # long comma-joined line. Valid JSON, but this file is read by people.

1241 local values_json="" line

1242 local remaining=${#value_lines[@]}

1243 for line in "${value_lines[@]}"; do

1244 remaining=$((remaining - 1))

1245 if [ "$remaining" -gt 0 ]; then

1246 values_json+="$line,"$'\n'

1247 else

1248 values_json+="$line"

1249 fi

1250 done

1251

1252 # The seed is written OUTSIDE the values block and unconditionally, because

1253 # it is no longer one of the required values (an absent --seed is randomized,

1254 # not refused) -- but recording it is the entire reason it exists. Its source

1255 # rides alongside so a reader can tell a chosen seed from a rolled one, which

1256 # is the distinction the record is for.

1257 cat > "$OUTPUT_DIR/generation-metadata.json" <<EOF

1258{

1259 "generated_at": "$generated_at",

1260 "stages": [$stages_json],

1261 "seed": $RANDOM_SEED,

1262 "seed_source": "$RANDOM_SEED_SOURCE",

1263 "values": {

1264$values_json

1265 }

1266}

1267EOF

1268}

1269# }}}

1270# }}}

1271

1272# {{{ Stage execution functions

1273

1274# {{{ run_update_words

1275run_update_words() {

1276 log_stage "📁 Stage 1/10: Updating input files from words repository"

1277

1278 # Issue 10-016: Check both global and per-stage force flags (Stage 1)

1279 local stage_force=$FORCE

1280 $FORCE_STAGE_1 && stage_force=true

1281

1282 # Issue 7-003: Pass force flag to skip file preservation

1283 local force_flag=""

1284 if $stage_force; then

1285 force_flag="--force"

1286 fi

1287

1288 if $DRY_RUN; then

1289 log_dry_run "$DIR/scripts/update-words $force_flag"

1290 return 0

1291 fi

1292

1293 # Issue 10-065: this was a warning that continued. It should not be. This

1294 # stage's whole job is to make input/ match the words repository, so a

1295 # failure means every stage after it reads a corpus that is not the one the

1296 # operator asked to publish -- and reads it without complaint, because the

1297 # files are all there, just old. A stale corpus is invisible in the output.

1298 "$DIR/scripts/update-words" $force_flag || {

1299 echo "Error: failed to sync input files from the words repository" >&2

1300 echo " Every later stage would read a stale corpus, and would not" >&2

1301 echo " be able to tell that it had. Fix the sync and re-run." >&2

1302 exit 1

1303 }

1304}

1305# }}}

1306

1307# {{{ run_extract

1308run_extract() {

1309 log_stage "🔄 Stage 2/10: Extracting content from backup archives"

1310

1311 # Issue 8-011/10-065: reshared-post inclusion, as an explicit yes or no. This

1312 # used to read a separate INCLUDE_BOOSTS boolean that the CLI could not

1313 # actually set (the flag's case branch was shadowed by an earlier duplicate),

1314 # so extraction silently used config.privacy.include_boosts no matter what

1315 # was typed. One required flag now feeds both this stage and parsing.

1316 local boost_flag="--no-boosts"

1317 if [ "$BOOSTS" = "yes" ]; then

1318 boost_flag="--include-boosts"

1319 fi

1320

1321 if $DRY_RUN; then

1322 log_dry_run "$DIR/scripts/update $DIR $boost_flag"

1323 return 0

1324 fi

1325

1326 "$DIR/scripts/update" "$DIR" $boost_flag || {

1327 echo "Error: Content extraction failed" >&2

1328 exit 1

1329 }

1330}

1331# }}}

1332

1333# {{{ run_strip_excluded

1334# Issue 10-053: After sync/extraction, remove excluded images + note source files

1335# from input/ so they are never cataloged, embedded, rendered, or uploaded. Runs

1336# before image cataloging. strip-excluded validates every exclusion BEFORE it

1337# deletes anything; a non-zero exit means a broken exclusion path (it points at no

1338# real file), which is FATAL -- continuing would ship content that was explicitly

1339# marked do-not-ship. The validation happens before any stripping and before the

1340# expensive catalog/embed stages, so a bad path costs only the cheap re-run.

1341run_strip_excluded() {

1342 log_stage "🧹 Stripping excluded content from input/"

1343 if $DRY_RUN; then

1344 log_dry_run "lua $DIR/scripts/strip-excluded $DIR"

1345 return 0

1346 fi

1347 if ! lua "$DIR/scripts/strip-excluded" "$DIR"; then

1348 echo "ERROR: strip-excluded failed -- a broken exclusion path in config.lua." >&2

1349 echo " Fix excluded_images and re-run; nothing was stripped or shipped." >&2

1350 exit 1

1351 fi

1352}

1353# }}}

1354

1355# {{{ run_parse

1356run_parse() {

1357 log_stage "📝 Stage 3/10: Parsing poems from JSON sources"

1358

1359 # Issue 10-016: Check both global and per-stage force flags (Stage 3)

1360 local stage_force=$FORCE

1361 $FORCE_STAGE_3 && stage_force=true

1362

1363 local force_arg=""

1364 if $stage_force; then

1365 force_arg="--force"

1366 fi

1367

1368 # Issue 10-065: the same explicit yes/no extraction used, so the two stages

1369 # cannot disagree about what a poem is. main.lua understands both spellings.

1370 local boosts_arg="--no-boosts"

1371 if [ "$BOOSTS" = "yes" ]; then

1372 boosts_arg="--include-boosts"

1373 fi

1374

1375 if $DRY_RUN; then

1376 log_dry_run "luajit src/main.lua $DIR --parse-only $force_arg $boosts_arg $ASSETS_ARG"

1377 return 0

1378 fi

1379

1380 luajit src/main.lua "$DIR" --parse-only $force_arg $boosts_arg $ASSETS_ARG || {

1381 echo "Error: Poem parsing failed" >&2

1382 exit 1

1383 }

1384}

1385# }}}

1386

1387# {{{ run_validate

1388run_validate() {

1389 log_stage "$(symbol_success "✓") Stage 4/10: Validating poem data"

1390

1391 if $DRY_RUN; then

1392 log_dry_run "luajit src/main.lua $DIR --validate-only $ASSETS_ARG"

1393 return 0

1394 fi

1395

1396 luajit src/main.lua "$DIR" --validate-only $ASSETS_ARG || {

1397 echo "Error: Poem validation failed" >&2

1398 exit 1

1399 }

1400}

1401# }}}

1402

1403# {{{ run_catalog_images

1404# Issue 10-015a: Pass --verbose flag to show detailed image catalog statistics

1405run_catalog_images() {

1406 log_stage "🖼️ Stage 5/10: Cataloging images"

1407

1408 # Build verbose argument if enabled

1409 local VERBOSE_ARG=""

1410 $VERBOSE && VERBOSE_ARG="--verbose"

1411

1412 if $DRY_RUN; then

1413 log_dry_run "luajit src/main.lua $DIR --catalog-only $VERBOSE_ARG $ASSETS_ARG $RANDOM_SEED_ARG"

1414 return 0

1415 fi

1416

1417 luajit src/main.lua "$DIR" --catalog-only $VERBOSE_ARG $ASSETS_ARG $RANDOM_SEED_ARG || {

1418 echo "Error: Image cataloging failed" >&2

1419 exit 1

1420 }

1421}

1422# }}}

1423

1424# {{{ emb_cache_dir

1425# Issue 10-054: resolve a model's cache directory through the shared resolver

1426# (scripts/cache-dir), so run.sh's freshness/pre-flight checks look in EXACTLY the

1427# place the Lua code and generate-embeddings.sh write -- disk or RAM, per the

1428# CACHE_IN_RAM switch. Pass --disk for the reboot-surviving diversity cache. A

1429# blank result is a hard error rather than a silently-wrong (empty) path.

1430emb_cache_dir() {

1431 local d

1432 d="$(luajit "$DIR/scripts/cache-dir" "$DIR" --model "$MODEL_NAME" "$@")"

1433 if [ -z "$d" ]; then

1434 echo "Error: could not resolve cache dir (scripts/cache-dir)" >&2

1435 exit 1

1436 fi

1437 echo "$d"

1438}

1439# }}}

1440

1441# {{{ require_embeddings_for_model()

1442# Issue 10-065 (open question 1): the check stages 7 and 8 already perform, given

1443# to stages 9 and 10, which lacked it.

1444#

1445# Why the disk is the right authority here. For stages 7-10 the model name is

1446# purely a CACHE DIRECTORY name -- they never contact the inference server. So

1447# the argument inference-server-config makes for refusing to validate --model

1448# (the server is the only authority on what is loaded, and a second authority

1449# drifts) simply does not reach them: there is no server in the loop to reject

1450# anything. What these stages actually need is embeddings on disk, and that is a

1451# thing this script can verify without keeping a second list in sync.

1452#

1453# What it prevents: a one-character slip. "--model qwen3-embedding:4B" resolves

1454# to a real, empty directory sitting beside the real ":4b" one, indistinguishable

1455# from a model whose embeddings have simply not been generated yet -- and stages

1456# 9 and 10 would then build a site against nothing.

1457#

1458# Called BEFORE each stage's dry-run check, matching stages 7 and 8: a dry run

1459# should say that the real run cannot work. The check only reads, so it is safe

1460# there.

1461require_embeddings_for_model() {

1462 local stage_label="$1"

1463 local embeddings_file

1464 embeddings_file="$(emb_cache_dir)/embeddings.json"

1465 if [ -f "$embeddings_file" ]; then

1466 return 0

1467 fi

1468

1469 echo "Error: no embeddings for model '$MODEL_NAME' (needed by $stage_label)" >&2

1470 echo " Looked for: $embeddings_file" >&2

1471

1472 # List what IS present, so a typo is visible by comparison. These are

1473 # DIRECTORY names, not model names: the mapping writes ':' as '_' and cannot

1474 # be reversed unambiguously (a model could contain a literal underscore), so

1475 # showing the directory and saying how it was spelled is the honest form.

1476 local root

1477 root="$(dirname "$(emb_cache_dir)")"

1478 local found=()

1479 local d

1480 for d in "$root"/*/; do

1481 [ -f "$d/embeddings.json" ] || continue

1482 found+=("$(basename "$d")")

1483 done

1484

1485 if [ "${#found[@]}" -gt 0 ]; then

1486 echo " Models with embeddings on disk (directory names; ':' is written '_'):" >&2

1487 local m

1488 for m in "${found[@]}"; do

1489 echo " $m" >&2

1490 done

1491 else

1492 echo " No model has embeddings yet. Run --generate-embeddings first." >&2

1493 fi

1494 exit 1

1495}

1496# }}}

1497

1498# {{{ run_generate_embeddings

1499run_generate_embeddings() {

1500 log_stage "🤖 Stage 6/10: Generating embeddings via the inference server"

1501

1502 # Convert model name for directory (embeddinggemma:latest -> embeddinggemma_latest)

1503 local model_dir_name="${MODEL_NAME//:/_}"

1504 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1505 # Issue 10-065: built from the resolved assets directory, not a hardcoded

1506 # "$DIR/assets". When --dir pointed the child programs at another corpus,

1507 # this freshness check went on reading the original one -- so the count it

1508 # compared against was from a different set of poems entirely.

1509 local poems_file="$ASSETS_DIR/poems.json"

1510

1511 # Issue 10-016: Check both global and per-stage force flags

1512 local stage_force=$FORCE

1513 $FORCE_STAGE_6 && stage_force=true

1514

1515 # Freshness check (Issue 10-050): skip ONLY when every poem already has an

1516 # embedding. The old test compared mtimes (embeddings.json newer than

1517 # poems.json) — which was wrong: a run that embedded 8160/8362 and then died

1518 # leaves a NEWER but INCOMPLETE embeddings.json, so mtime said "fresh, skip"

1519 # and the missing poems never got done. Counting entries is the honest

1520 # signal; incremental mode then fills only the gap, so it is cheap to re-run.

1521 if ! $stage_force && [ -f "$embeddings_file" ] && [ -f "$poems_file" ]; then

1522 # Count embeddings WITHOUT parsing the (large) JSON: each entry carries

1523 # exactly one "poem_index" key. (This counts error records too, so it can

1524 # only over-report completeness; incremental retries those anyway.)

1525 local emb_count

1526 emb_count=$(grep -o '"poem_index"' "$embeddings_file" | wc -l)

1527 local poem_count

1528 # Issue 10-065: `#(d.poems or d)` used to stand here -- "the poems field,

1529 # or else treat the whole document as the list". That absorbed a change

1530 # in the file's shape instead of reporting one, and the consequence was

1531 # not an error but a NUMBER: a wrong count, silently compared against the

1532 # embeddings, deciding whether to skip a 2-3 hour stage.

1533 poem_count=$(luajit -e "

1534 package.path = '$DIR/?.lua;' .. package.path

1535 local dk = require('libs/dkjson')

1536 local f = io.open('$poems_file')

1537 if not f then error('cannot open $poems_file') end

1538 local d = dk.decode(f:read('*a')); f:close()

1539 if type(d) ~= 'table' or type(d.poems) ~= 'table' then

1540 error('$poems_file has no poems array; cannot count the corpus')

1541 end

1542 print(#d.poems)

1543 ")

1544 if [ -z "$poem_count" ]; then

1545 echo "Error: could not count poems in $poems_file" >&2

1546 echo " Without a corpus size there is no way to tell a complete" >&2

1547 echo " embeddings file from an interrupted one." >&2

1548 exit 1

1549 fi

1550 if [ "$poem_count" -gt 0 ] && [ "$emb_count" -ge "$poem_count" ]; then

1551 log_info " ⏭️ Embeddings complete ($emb_count/$poem_count), skipping..."

1552 return 0

1553 fi

1554 log_info " Embeddings incomplete ($emb_count/$poem_count) — running incremental to fill the gap..."

1555 fi

1556

1557 local force_arg=""

1558 if $stage_force; then

1559 force_arg="--full-regen"

1560 else

1561 force_arg="--incremental"

1562 fi

1563

1564 # Issue 10-017/10-065: the server is required for this stage, so the flag is

1565 # built unconditionally. It used to be omitted when INFERENCE_SERVER was

1566 # empty, which handed the choice to config.default_inference_server -- and

1567 # the log line below was likewise conditional, so the run did not even say

1568 # which endpoint it had spent the whole stage talking to.

1569 local server_arg="--server=$INFERENCE_SERVER"

1570

1571 if $DRY_RUN; then

1572 log_dry_run "$DIR/generate-embeddings.sh $force_arg --model=$MODEL_NAME $server_arg $DIR"

1573 log_dry_run "luajit $DIR/src/generate-word-pages.lua $DIR --embeddings-only $ASSETS_ARG"

1574 return 0

1575 fi

1576

1577 log_info " Inference Server: $INFERENCE_SERVER"

1578 log_info " Model: $MODEL_NAME"

1579 log_info " Output: assets/embeddings/$model_dir_name/embeddings.json"

1580 log_info " Mode: $(if $FORCE; then echo 'full regeneration'; else echo 'incremental (skip existing)'; fi)"

1581

1582 # Issue 10-028: Apply low priority to expensive embedding generation

1583 $NICE_PREFIX "$DIR/generate-embeddings.sh" $force_arg --model="$MODEL_NAME" $server_arg "$DIR" || {

1584 echo "Error: Embedding generation failed" >&2

1585 echo "Make sure the inference server is running with the $MODEL_NAME model" >&2

1586 exit 1

1587 }

1588

1589 # Word embeddings used to run here, but the word-COLOR step inside

1590 # generate-word-pages needs color_embeddings.json, which is produced later by

1591 # run_generate_semantic_colors. Running words first made that step skip with

1592 # "no color embeddings found". Moved to run_generate_word_embeddings, called

1593 # AFTER colors in main.

1594}

1595# }}}

1596

1597# {{{ run_generate_word_embeddings

1598# Word-cloud word embeddings + their semantic colors. Split out of

1599# run_generate_embeddings (Issue 8-043b) and ordered AFTER the semantic-color

1600# stage so color_embeddings.json already exists when the word-color step runs.

1601run_generate_word_embeddings() {

1602 log_info " Generating word embeddings for word cloud..."

1603 # WORDCLOUD_WORDS carries either a number or the literal "all"; the generator

1604 # accepts both via --words (it treats "--words all" the same as "--all").

1605 # Issue 10-065: passed unconditionally -- the value is required by this stage,

1606 # so there is no "omit the flag and let the generator pick a count" case.

1607 local wordcloud_args="--words $WORDCLOUD_WORDS"

1608

1609 # Issue 10-065: this was the ONLY stage function with no dry-run guard, found

1610 # by auditing all fourteen of them after two others turned up with the guard

1611 # in the wrong PLACE. It is the worst of the three: the others touched local

1612 # files, while this one opened a network connection to the inference server

1613 # and began embedding -- so `--dry-run` did real work on another machine.

1614 if $DRY_RUN; then

1615 log_dry_run "luajit $DIR/src/generate-word-pages.lua $DIR --embeddings-only $wordcloud_args $ASSETS_ARG"

1616 return 0

1617 fi

1618

1619 # Issue 10-065: was a warning that continued. Word embeddings are what the

1620 # per-word similarity pages rank against, so continuing meant stage 10 would

1621 # later build its pages from whatever stale embeddings were lying around --

1622 # and produce a complete-looking word cloud whose rankings answer a question

1623 # nobody asked on this run.

1624 $NICE_PREFIX luajit "$DIR/src/generate-word-pages.lua" "$DIR" --embeddings-only $wordcloud_args $ASSETS_ARG || {

1625 echo "Error: word embedding generation failed" >&2

1626 echo " The per-word similarity pages rank against these embeddings;" >&2

1627 echo " continuing would build stage 10 from stale ones." >&2

1628 exit 1

1629 }

1630}

1631# }}}

1632

1633# {{{ run_generate_semantic_colors

1634run_generate_semantic_colors() {

1635 # Regenerate poem_colors.json if stale or missing

1636 # This runs BEFORE similarity matrix generation (Stage 6.5)

1637 # Requires: embeddings.json, color_embeddings.json

1638 # Respects: --force (skip freshness check), --dry-run (show actions only)

1639

1640 local model_dir_name="${MODEL_NAME//:/_}"

1641

1642 # Paths match what generate-embeddings.sh writes (see run_generate_embeddings above).

1643 # The stray assets/embeddings/embeddings/ directory on disk is a stale leftover from

1644 # before the model-name subfolder convention; it is not the real output location.

1645 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1646 local poem_colors_file="$(emb_cache_dir)/poem_colors.json"

1647 local color_embeddings_file="$(emb_cache_dir)/color_embeddings.json"

1648

1649 # Issue 10-065: this used to log at VERBOSE level and return success. That is

1650 # the quietest failure in the script: this function only ever runs as part of

1651 # stage 6, immediately after the stage that writes embeddings.json -- so the

1652 # file being absent means that stage did not do its job, and the run reported

1653 # a clean pipeline having skipped a stage. At default verbosity the message

1654 # was not even printed.

1655 if [ ! -f "$embeddings_file" ]; then

1656 echo "Error: embeddings not found: $embeddings_file" >&2

1657 echo " Semantic colours run as part of stage 6, right after the step" >&2

1658 echo " that writes this file, so its absence means that step failed." >&2

1659 exit 1

1660 fi

1661

1662 # color_embeddings.json is derived from the color palette (color_names +

1663 # color_associations in config.lua). It used to regenerate ONLY when the file was

1664 # missing, so editing the palette -- e.g. dropping gray as a cluster color -- had

1665 # no effect until someone deleted the cache by hand (and the config comment that

1666 # said "re-run stage 6.5 after editing" was quietly false). We now fingerprint the

1667 # palette and regenerate whenever it changes, so editing colors then re-running

1668 # actually takes effect. The fingerprint is a sorted, deterministic dump of the

1669 # palette -- no server needed to compute it.

1670 local palette_fp_file="$(emb_cache_dir)/color_palette.fingerprint"

1671 local current_palette_fp

1672 # Issue 10-065: the empty-table stand-ins here (`config.color_names or {}`,

1673 # `(config.color_associations or {})[n] or {}`) were quietly load-bearing. A

1674 # config with no palette fingerprinted to the SAME empty string as a config

1675 # whose palette had simply not changed -- so a broken config read as "still

1676 # fresh, skip", and the colour cache was never rebuilt. A fingerprint whose

1677 # failure mode is a valid-looking value is worse than no fingerprint.

1678 current_palette_fp=$(luajit -e "

1679 package.path = '$DIR/libs/?.lua;$DIR/src/?.lua;' .. package.path

1680 local config = require('config-loader').load()

1681 if type(config.color_names) ~= 'table' or #config.color_names == 0 then

1682 error('config.lua has no color_names; the semantic-colour palette is empty')

1683 end

1684 if type(config.color_associations) ~= 'table' then

1685 error('config.lua has no color_associations table')

1686 end

1687 local names = {}

1688 for _, n in ipairs(config.color_names) do names[#names+1] = n end

1689 table.sort(names)

1690 local parts = {}

1691 for _, n in ipairs(names) do

1692 local assoc = config.color_associations[n]

1693 if type(assoc) ~= 'table' then

1694 error('color_associations has no entry for the colour ' .. n)

1695 end

1696 local a = {}

1697 for _, w in ipairs(assoc) do a[#a+1] = w end

1698 table.sort(a)

1699 parts[#parts+1] = n .. '=' .. table.concat(a, ',')

1700 end

1701 io.write(table.concat(parts, '|'))

1702 ") || {

1703 echo "Error: could not fingerprint the colour palette from config.lua" >&2

1704 exit 1

1705 }

1706 if [ -z "$current_palette_fp" ]; then

1707 echo "Error: colour palette fingerprint came back empty" >&2

1708 exit 1

1709 fi

1710 local stored_palette_fp=""

1711 [ -f "$palette_fp_file" ] && stored_palette_fp=$(cat "$palette_fp_file")

1712

1713 # Regenerate color embeddings if missing OR the palette changed since last time.

1714 if [ ! -f "$color_embeddings_file" ] || [ "$current_palette_fp" != "$stored_palette_fp" ]; then

1715 if [ -f "$color_embeddings_file" ]; then

1716 log_stage "🎨 Stage 6.5/10: Color palette changed -- regenerating color embeddings"

1717 else

1718 log_stage "🎨 Stage 6.5/10: Generating color embeddings (one-time)"

1719 fi

1720

1721 if $DRY_RUN; then

1722 log_dry_run "luajit semantic-color-calculator (generate color embeddings)"

1723 # Still need to skip poem colors generation in dry run

1724 else

1725 log_info " $(symbol_warning "⚠️") Color embeddings not found, generating via the inference server..."

1726 # Issue 10-003 migrated color_names from config/semantic-colors.json (now deleted)

1727 # into config.lua, loaded via libs/config-loader.lua. Errors here are loud rather

1728 # than silent so a missing config doesn't propagate downstream as a confusing

1729 # "Failed to load required data files" in the next stage.

1730 luajit -e "

1731 package.path = '$DIR/libs/?.lua;$DIR/src/?.lua;' .. package.path

1732 local calc = require('semantic-color-calculator')

1733 local utils = require('utils')

1734 -- Issue 10-065: '--dir', PATH -- not a bare positional. This read

1735 -- init_assets_root({'\$DIR'}), and utils.parse_assets_dir only

1736 -- recognizes '--dir PATH' or '--dir=PATH', so a lone positional

1737 -- was DISCARDED and the block fell through to the project's own

1738 -- assets directory. It happened to be the same path, which is why

1739 -- it went unnoticed -- until --dir pointed somewhere else, and

1740 -- then this block silently read a different corpus than the

1741 -- stage that launched it.

1742 utils.init_assets_root({'--dir', '$ASSETS_DIR'})

1743

1744 -- Issue 10-065: the server is set unconditionally now. It used to

1745 -- be guarded by a non-empty test on INFERENCE_SERVER, and an

1746 -- empty one meant the module quietly used config.lua's

1747 -- default_inference_server -- so a colour palette could be

1748 -- embedded against a different endpoint than the poems were.

1749 -- The interactive flag is forwarded so that a typoed --server or

1750 -- --model triggers a 1/2 prompt only when the operator launched

1751 -- run.sh with -I; otherwise we hard-error.

1752 local inference = require('inference-server-config')

1753 inference.set_project_root('$DIR')

1754 inference.set_interactive_mode('$INTERACTIVE' == 'true')

1755 inference.set_selected_server('$INFERENCE_SERVER')

1756

1757 local config = require('config-loader').load()

1758 if not config.color_names then

1759 error('config.lua is missing color_names (Issue 10-003 migration)')

1760 end

1761 if type(config.color_associations) ~= 'table' then

1762 error('config.lua is missing color_associations; each colour embedding '

1763 .. 'is the mean of its essence words, and without them the '

1764 .. 'calculator would silently embed the bare colour word instead')

1765 end

1766 -- Pass color_associations so each color's embedding is the mean

1767 -- of its essence words, not the bare color word (richer + the

1768 -- z-scored assignment is balanced). nil endpoint = use the

1769 -- selected server. Issue 10-065 added the check above: the

1770 -- calculator's own bare-word fallback is still there, but it can

1771 -- no longer be reached from here without anyone noticing.

1772 local embeddings = calc.generate_color_embeddings(config.color_names, '$MODEL_NAME', nil, config.color_associations)

1773 if not next(embeddings) then

1774 error('Inference server returned no color embeddings')

1775 end

1776 local data = {embeddings = embeddings, generated_at = os.date('%Y-%m-%d %H:%M:%S'), model_name = '$MODEL_NAME'}

1777 utils.write_json_file('$color_embeddings_file', data)

1778 print('[INFO] Color embeddings saved: ' .. '$color_embeddings_file')

1779 " || {

1780 echo "Error: Color embedding generation failed" >&2

1781 exit 1

1782 }

1783 # Remember the palette we just built from, so the next run can tell

1784 # whether it changed (and skip this server round-trip when it hasn't).

1785 echo "$current_palette_fp" > "$palette_fp_file"

1786 fi

1787 fi

1788

1789 # Issue 10-016: Check both global and per-stage force flags (Stage 6)

1790 local stage_force=$FORCE

1791 $FORCE_STAGE_6 && stage_force=true

1792

1793 # Check freshness: poem_colors.json should be newer than embeddings.json

1794 # With --force or --force-stage 6: always regenerate regardless of freshness

1795 if ! $stage_force && [ -f "$poem_colors_file" ] && [ -f "$embeddings_file" ]; then

1796 # Poem colors depend on BOTH the poem embeddings AND the color centroids, so

1797 # they are only fresh when newer than both. Watching only embeddings.json

1798 # meant a palette change (which rewrites color_embeddings.json but not

1799 # embeddings.json) left poem_colors.json stale yet considered "fresh".

1800 if [ "$poem_colors_file" -nt "$embeddings_file" ] && [ "$poem_colors_file" -nt "$color_embeddings_file" ]; then

1801 log_info " ⏭️ Semantic colors are fresh (newer than embeddings + palette), skipping..."

1802 return 0

1803 fi

1804 log_verbose " poem_colors.json is stale (older than embeddings or palette), regenerating..."

1805 elif $stage_force; then

1806 log_verbose " --force specified, regenerating semantic colors..."

1807 fi

1808

1809 log_stage "🎨 Stage 6b/10: Computing semantic colors (part of embeddings)"

1810

1811 if $DRY_RUN; then

1812 log_dry_run "luajit semantic-color-calculator (poem colors regeneration)"

1813 return 0

1814 fi

1815

1816 log_info " Input: $embeddings_file"

1817 log_info " Output: $poem_colors_file"

1818

1819 # Regenerate poem colors using existing embeddings

1820 luajit -e "

1821 package.path = '$DIR/libs/?.lua;$DIR/src/?.lua;' .. package.path

1822 local calc = require('semantic-color-calculator')

1823 local utils = require('utils')

1824 utils.init_assets_root({'--dir', '$ASSETS_DIR'})

1825

1826 local poems_data = utils.read_json_file(utils.asset_path('poems.json'))

1827 local embeddings_data = utils.read_json_file('$embeddings_file')

1828 local color_embeddings_data = utils.read_json_file('$color_embeddings_file')

1829

1830 if poems_data and embeddings_data and color_embeddings_data then

1831 calc.precompute_poem_colors(poems_data, embeddings_data, color_embeddings_data.embeddings, '$poem_colors_file')

1832 else

1833 error('Failed to load required data files')

1834 end

1835 " || {

1836 echo "Error: Semantic color generation failed" >&2

1837 exit 1

1838 }

1839}

1840# }}}

1841

1842# {{{ run_augment_images

1843# Issue 9-013: give every text-less image a pseudo-embedding (the normalized

1844# average of the poem before and after it chronologically) and fold those into

1845# embeddings.json so the GPU similarity stage ranks images alongside poems.

1846# Also writes image-manifest.json, which the HTML renderer reads to draw image

1847# entries. Cheap and idempotent, so it runs each time before the matrix build.

1848run_augment_images() {

1849 log_stage "🖼️ Stage 6.7: Folding images into the embedding set (pseudo-embeddings)"

1850 local model_dir_name="${MODEL_NAME//:/_}"

1851 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1852 if [ ! -f "$embeddings_file" ]; then

1853 echo "Error: embeddings.json not found; run --generate-embeddings first" >&2

1854 exit 1

1855 fi

1856 if $DRY_RUN; then

1857 log_dry_run "luajit $DIR/src/augment-embeddings-with-images.lua $DIR"

1858 return 0

1859 fi

1860 $NICE_PREFIX luajit "$DIR/src/augment-embeddings-with-images.lua" "$DIR" || {

1861 echo "Error: image augmentation failed" >&2

1862 exit 1

1863 }

1864}

1865# }}}

1866

1867# {{{ run_generate_similarity

1868run_generate_similarity() {

1869 # GPU (Vulkan) is required: these are O(N^2) similarity calculations that make no

1870 # sense on a CPU, so the CPU route was removed (Issue 10-057). A missing GPU library

1871 # is a hard error with build instructions, never a slow fallback.

1872 if [ ! -f "$DIR/libs/vulkan-compute/build/libvkcompute.so" ]; then

1873 echo "Error: GPU library not found: libs/vulkan-compute/build/libvkcompute.so" >&2

1874 echo "Build it: cd libs/vulkan-compute && make" >&2

1875 exit 1

1876 fi

1877 log_stage "📊 Stage 7/10: Building similarity matrix with GPU"

1878

1879 # Convert model name for directory

1880 local model_dir_name="${MODEL_NAME//:/_}"

1881 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1882

1883 # Check if embeddings exist

1884 if [ ! -f "$embeddings_file" ]; then

1885 echo "Error: Embeddings file not found: $embeddings_file" >&2

1886 echo "Run --generate-embeddings first" >&2

1887 exit 1

1888 fi

1889

1890 # Issue 10-016: Check both global and per-stage force flags (Stage 7)

1891 local stage_force=$FORCE

1892 $FORCE_STAGE_7 && stage_force=true

1893

1894 # Issue 8-033: Check for individual similarity files instead of monolithic matrix

1895 local similarities_dir="$(emb_cache_dir)/similarities"

1896 local similarity_count=0

1897 if [ -d "$similarities_dir" ]; then

1898 # Issue 10-065: no 2>/dev/null here. The directory's existence is already

1899 # checked above, so anything find still complains about (an unreadable

1900 # subdirectory, say) would make this count too LOW -- and a count that is

1901 # too low reads as "cache incomplete, rebuild it", quietly turning a

1902 # permissions problem into a multi-hour regeneration.

1903 similarity_count=$(find "$similarities_dir" -name "poem_*.json" | wc -l)

1904 fi

1905

1906 # How many similarity files SHOULD exist: exactly one per embedding. The

1907 # embeddings file is the only thing that knows, so it is what we ask.

1908 #

1909 # Issue 10-065: this used to be the literal 7797. Two things were wrong with

1910 # it, and the second is why a number could not be made to work here at all.

1911 #

1912 # It was already stale. The real count is one per EMBEDDING, and the

1913 # embedding set includes the image pseudo-embeddings folded in by stage 6.7

1914 # (Issue 9-013) -- 8701 against 8050 poems when this was written. So a run

1915 # that died after 7900 files cleared a 7797 threshold and was called

1916 # complete, leaving ~800 entries with no similarity data for the HTML stage

1917 # to build pages from.

1918 #

1919 # It goes stale continuously. The corpus grows every time a poem is

1920 # written, so any constant here is correct only until the next poem.

1921 #

1922 # This is the same defect Issue 10-050 fixed one stage earlier, where an

1923 # mtime comparison called an interrupted embeddings.json fresh. The lesson it

1924 # recorded -- "counting entries is the honest signal" -- simply had not been

1925 # carried into stage 7. Counting the embeddings costs one grep over a file

1926 # this stage is about to read in full anyway.

1927 local expected_count

1928 expected_count=$(grep -o '"poem_index"' "$embeddings_file" | wc -l)

1929 if [ -z "$expected_count" ] || [ "$expected_count" -eq 0 ]; then

1930 echo "Error: $embeddings_file contains no embeddings" >&2

1931 echo " Without a count there is no way to tell a complete similarity" >&2

1932 echo " cache from an interrupted one. Re-run --generate-embeddings." >&2

1933 exit 1

1934 fi

1935

1936 # Freshness check: skip only when every embedding has a similarity file AND

1937 # those files are newer than the embeddings they were computed from.

1938 if ! $stage_force && [ "$similarity_count" -ge "$expected_count" ]; then

1939 # Check if any are older than embeddings (check newest file)

1940 local newest_similarity=$(find "$similarities_dir" -name "poem_*.json" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-)

1941 if [ -n "$newest_similarity" ] && [ "$newest_similarity" -nt "$embeddings_file" ]; then

1942 log_info " ⏭️ Similarity files are fresh ($similarity_count/$expected_count, newer than embeddings), skipping..."

1943 return 0

1944 fi

1945 elif ! $stage_force && [ "$similarity_count" -gt 0 ]; then

1946 # Say the count out loud. An interrupted cache used to be indistinguishable

1947 # from a complete one in the log; now the shortfall is on screen.

1948 log_info " Similarity cache incomplete ($similarity_count/$expected_count) — rebuilding..."

1949 fi

1950

1951 # Issue 10-065: unconditional. --threads is required by this stage.

1952 local threads_arg="--threads=$THREADS"

1953

1954 if $DRY_RUN; then

1955 log_dry_run "luajit (GPU vk_similarity via libvkcompute.so) --generate-matrix $threads_arg"

1956 return 0

1957 fi

1958

1959 log_info " Input: assets/embeddings/$model_dir_name/embeddings.json"

1960 log_info " Output: assets/embeddings/$model_dir_name/similarities/*.json (individual files)"

1961

1962 # Issue 10-016: Convert stage_force to Lua boolean for Lua function calls

1963 local stage_force_lua="false"

1964 $stage_force && stage_force_lua="true"

1965

1966 # GPU similarity generation using Vulkan compute shaders (the only route now)

1967 log_info " Mode: GPU-accelerated (Vulkan)"

1968

1969 # Issue 10-065: the literal 8 that used to stand in here is gone. It was

1970 # the most concrete example of the problem: a number written into the

1971 # source of one stage, disagreeing with the "(default: 4)" the help text

1972 # advertised, and with whatever the HTML stage's child picked for itself.

1973 log_info " CPU sorting threads: $THREADS"

1974

1975 DIR="$DIR" luajit -e "

1976 package.path = '$DIR/?.lua;$DIR/?/init.lua;$DIR/libs/?.lua;' .. package.path

1977 local vk_sim = require('libs.vulkan-compute.lua.vk_similarity')

1978 -- Issue 10-057: size the rankings cache to exactly what THIS build shows

1979 -- per poem -- the ACTUAL pages it generates times the poems shown per

1980 -- page -- NOT the storage ceiling max_pages_per_poem. The list is sorted

1981 -- nearest-first, so the top-K ARE precisely what the pages display. The

1982 -- HTML stage's loader regenerates if a later run needs more (the top_k

1983 -- stamp makes that detectable).

1984 --

1985 -- Issue 10-065: both numbers come from the command line and nowhere

1986 -- else. They used to read 'the --pages value, or else

1987 -- config.pagination.minimum_pages' -- and that mattered more here

1988 -- than it looks, because THIS number decides how much of the

1989 -- similarity cache gets built, while stage 9 separately decides how

1990 -- much of it to display. Two stages resolving the same value through

1991 -- different fallbacks is how a cache ends up one page short of the

1992 -- pages that read it.

1993 local _pages = tonumber('$PAGES')

1994 local _per_page = tonumber('$POEMS_PER_PAGE')

1995 if not _pages then error('--pages is not a number: $PAGES') end

1996 if not _per_page then error('--poems-per-page is not a number: $POEMS_PER_PAGE') end

1997 local _top_k = _pages * _per_page

1998 -- Use TRUE parallel GPU computation (Issue 9-002 original design)

1999 local success = vk_sim.generate_similarity_matrix_gpu_parallel(

2000 '$(emb_cache_dir)/embeddings.json',

2001 '$MODEL_NAME',

2002 $stage_force_lua,

2003 $THREADS,

2004 _top_k

2005 )

2006 if not success then

2007 print('[GPU SIMILARITY ERROR] GPU generation failed')

2008 os.exit(1)

2009 end

2010 " || {

2011 echo "Error: GPU similarity generation failed" >&2

2012 exit 1

2013 }

2014

2015 # Note: Pre-sorted similarity rankings cache is now generated automatically

2016 # by the GPU similarity engine (in-RAM, no file re-reading needed)

2017}

2018# }}}

2019

2020# {{{ run_generate_diversity

2021run_generate_diversity() {

2022 # GPU (Vulkan) is required: the diversity walk is O(N^2) GPU work, so the CPU route

2023 # was removed (Issue 10-057). A missing GPU library is a hard error, not a fallback.

2024 if [ ! -f "$DIR/libs/vulkan-compute/build/libvkcompute.so" ]; then

2025 echo "Error: GPU library not found: libs/vulkan-compute/build/libvkcompute.so" >&2

2026 echo "Build it: cd libs/vulkan-compute && make" >&2

2027 exit 1

2028 fi

2029 log_stage "🎲 Stage 8/10: Pre-computing diversity cache with GPU"

2030

2031 # Convert model name for directory

2032 local model_dir_name="${MODEL_NAME//:/_}"

2033 local cache_file="$(emb_cache_dir --disk)/diversity_cache.json"

2034 local embeddings_file="$(emb_cache_dir)/embeddings.json"

2035

2036 # Check if embeddings exist

2037 if [ ! -f "$embeddings_file" ]; then

2038 echo "Error: Embeddings file not found: $embeddings_file" >&2

2039 echo "Run --generate-embeddings first" >&2

2040 exit 1

2041 fi

2042

2043 # Issue 10-016: Check both global and per-stage force flags (Stage 8)

2044 local stage_force=$FORCE

2045 $FORCE_STAGE_8 && stage_force=true

2046

2047 # Freshness check: skip if cache newer than embeddings

2048 if ! $stage_force && [ -f "$cache_file" ]; then

2049 if [ "$cache_file" -nt "$embeddings_file" ]; then

2050 log_info " ⏭️ Diversity cache is fresh (newer than embeddings), skipping..."

2051 return 0

2052 fi

2053 fi

2054

2055 log_info " Input: assets/embeddings/$model_dir_name/embeddings.json"

2056 log_info " Output: assets/embeddings/$model_dir_name/diversity_cache.json"

2057

2058 # GPU diversity generation using Vulkan compute shaders (the only route now)

2059 log_info " Mode: GPU-accelerated (Vulkan)"

2060

2061 if $DRY_RUN; then

2062 log_dry_run "$DIR/scripts/precompute-diversity-sequences-gpu $DIR"

2063 return 0

2064 fi

2065

2066 # Issue 10-028: Apply low priority to expensive diversity generation.

2067 # The model is not passed via env here: the wrapper resolves it through

2068 # inference-server-config, which reads this run's overrides notepad

2069 # (tmp/shared-memory/run-overrides.lua). Issue 10-065: that notepad now

2070 # always carries a model, because --model is required -- so the sentence

2071 # that used to end this comment ("and falls back to config.lua") no

2072 # longer describes anything that can happen.

2073 # Issue 10-057: pass the run's page settings so the wrapper caps each diversity

2074 # sequence to the SAME K the similarity cache and the HTML stage use.

2075 PAGES="$PAGES" POEMS_PER_PAGE="$POEMS_PER_PAGE" $NICE_PREFIX "$DIR/scripts/precompute-diversity-sequences-gpu" "$DIR" || {

2076 echo "Error: GPU diversity cache generation failed" >&2

2077 exit 1

2078 }

2079}

2080# }}}

2081

2082# {{{ run_generate_html

2083run_generate_html() {

2084 log_stage "🌐 Stage 9/10: Generating website HTML"

2085

2086 # Issue 10-065: this stage reads the model's caches (through the run-overrides

2087 # notepad) but never contacts a server, so nothing else here would catch a

2088 # mistyped --model. See require_embeddings_for_model.

2089 require_embeddings_for_model "stage 9 (generate-html)"

2090

2091 # Issue 10-016: Check both global and per-stage force flags (Stage 9)

2092 local stage_force=$FORCE

2093 $FORCE_STAGE_9 && stage_force=true

2094

2095 local force_arg=""

2096 if $stage_force; then

2097 force_arg="--force"

2098 fi

2099

2100 # Issue 10-065: every one of these was a conditional that omitted the flag

2101 # when the value was empty, handing the decision to the child program. Four

2102 # separate places where the command line could quietly stop mattering. All

2103 # four values are required by this stage, so all four are always passed --

2104 # which also means the --dry-run output below now shows the complete,

2105 # runnable command rather than an abridged one.

2106 local threads_arg="--threads $THREADS"

2107 local pages_arg="--pages $PAGES"

2108 local poems_per_page_arg="--poems-per-page $POEMS_PER_PAGE"

2109 local chrono_per_page_arg="--chrono-per-page $CHRONO_PER_PAGE"

2110

2111 # Same ordering rule as the word-cloud stage: the dry-run check comes before

2112 # the clear below, because --dry-run must not delete anything. This stage's

2113 # clear is guarded by --force, so it took BOTH flags to trigger -- which made

2114 # it rarer than the word-cloud one and no less destructive when it fired.

2115 if $DRY_RUN; then

2116 if $stage_force; then

2117 log_dry_run "rm $OUTPUT_DIR/{similar,different,chronological}/*.html (clear stale pages, --force)"

2118 fi

2119 log_dry_run "$DIR/scripts/sync-page-templates $DIR (restore explore-page copy into input/pages/)"

2120 log_dry_run "luajit src/main.lua $DIR --html-only $force_arg $threads_arg $pages_arg $poems_per_page_arg $chrono_per_page_arg $ASSETS_ARG"

2121 log_dry_run "luajit $DIR/src/generate-gallery-pages.lua $DIR $ASSETS_ARG"

2122 log_dry_run "luajit $DIR/src/generate-source-browser.lua $DIR"

2123 return 0

2124 fi

2125

2126 # Issue 10-024: Clear output directories when forcing regeneration.

2127 # This prevents stale files with obsolete poem_index values from persisting

2128 # after poem re-extraction changes the poem_index assignments.

2129 # Issue 10-065: the 2>/dev/null that used to hide these was suppressing the

2130 # one thing worth knowing -- that the clear did not happen. A glob matching

2131 # nothing is fine and silent anyway; a permission error is not.

2132 if $stage_force; then

2133 log_info " Clearing stale HTML files (--force)..."

2134 rm -f "$OUTPUT_DIR/similar/"*.html

2135 rm -f "$OUTPUT_DIR/different/"*.html

2136 rm -f "$OUTPUT_DIR/chronological/"*.html

2137 fi

2138

2139 # Issue 11-005: restore the authored explore-page copy into the ephemeral

2140 # input/pages/ before generating. The canonical, version-controlled source is

2141 # page-templates/*.txt; input/ is wiped + re-synced from external sources each

2142 # run and does NOT carry this prose, so it is copied back in here. (Edit the

2143 # files in page-templates/ -- input/pages/ is overwritten from them.)

2144 "$DIR/scripts/sync-page-templates" "$DIR" || {

2145 echo "Error: failed to restore page templates into input/pages/" >&2

2146 exit 1

2147 }

2148

2149 # Issue 10-028: Apply low priority to HTML generation (parallel processing)

2150 $NICE_PREFIX luajit src/main.lua "$DIR" --html-only $force_arg $threads_arg $pages_arg $poems_per_page_arg $chrono_per_page_arg $ASSETS_ARG || {

2151 echo "Error: HTML generation failed" >&2

2152 exit 1

2153 }

2154

2155 # Issue 10-059: the word-cloud menu and per-word similarity pages moved to their

2156 # own stage 10 (run_generate_wordcloud). They run after this stage, so the

2157 # chronological pages main.lua just built are already present for their #poem links.

2158

2159 # Issue 10-042: Build the image gallery (masonry pages per source + index +

2160 # chronological). It was previously a separate manual step, so the gallery

2161 # went stale -- it now regenerates with every HTML run from image-catalog.json.

2162 # Issue 10-065: was a warning that continued. The gallery is linked from the

2163 # site's navigation whether or not it built, so continuing publishes a page

2164 # of broken links -- and the pipeline's final line still reads "completed

2165 # successfully", which is the part that makes it hard to catch.

2166 log_info " Generating image gallery..."

2167 $NICE_PREFIX luajit "$DIR/src/generate-gallery-pages.lua" "$DIR" $ASSETS_ARG || {

2168 echo "Error: image gallery generation failed" >&2

2169 echo " The site links to the gallery either way, so continuing" >&2

2170 echo " would publish those links pointing at nothing." >&2

2171 exit 1

2172 }

2173

2174 # Issue 10-052: Build the link-only source browser (code/issues/docs as HTML)

2175 # under output/source/. This is the "git push that builds a webpage" -- the

2176 # private monorepo never leaves the machine; whoever has the site link can

2177 # browse the source. It publishes an ALLOWLIST only (never the private input

2178 # corpus), so it is safe to ship with the rest of the site.

2179 # Issue 10-065: was a warning that continued. This one publishes source code

2180 # against an allowlist, so a partial run is not merely incomplete -- a

2181 # half-finished pass is the wrong thing to be relaxed about when the

2182 # question it answers is "which files leave this machine".

2183 log_info " Generating source browser..."

2184 $NICE_PREFIX luajit "$DIR/src/generate-source-browser.lua" "$DIR" || {

2185 echo "Error: source browser generation failed" >&2

2186 echo " This stage decides which source files are published, so a" >&2

2187 echo " partial run is not something to continue past." >&2

2188 exit 1

2189 }

2190 # NOTE: the downloadable zip is built at POST time by running

2191 # scripts/build-download-zip directly, not here -- it is a deploy artifact, and

2192 # there is no point regenerating a multi-GB archive on every local build. (The

2193 # site's links are document-relative, so there is no URL-conversion step before

2194 # upload; just upload output/ and build the zip.)

2195}

2196# }}}

2197

2198# {{{ run_generate_wordcloud

2199# Issue 10-059: the word-cloud stage. Builds the site's entry menu (which carries the

2200# live poem index) and the per-word similarity pages. Runs after stage 9, so the

2201# chronological pages its #poem links target already exist. Replaces the retired

2202# numeric-similarity-index stage, whose output (numeric-index.html) was linked from

2203# nowhere and was superseded by the menu's embedded poem index.

2204run_generate_wordcloud() {

2205 log_stage "🔤 Stage 10/10: Generating word-cloud menu and per-word pages"

2206

2207 # Issue 10-065: same reasoning as stage 9. The per-word similarity pages rank

2208 # against this model's word embeddings, so a mistyped --model would rank

2209 # against an empty directory. See require_embeddings_for_model.

2210 require_embeddings_for_model "stage 10 (generate-wordcloud)"

2211

2212 # Word-cloud arguments. WORDCLOUD_WORDS is a number or "all"; --words carries

2213 # either ("--words all" == every word, per the generators).

2214 # Issue 10-065: all three passed unconditionally. The chrono_per_page one is

2215 # the instructive case -- it exists (Issue 10-036) precisely so this stage

2216 # and stage 9 agree on how many poems fit a chronological page, and yet it

2217 # was OMITTED when empty, letting the two stages resolve it independently.

2218 # A flag whose entire purpose is agreement cannot be optional.

2219 local wordcloud_words_arg="--words $WORDCLOUD_WORDS"

2220 local wordcloud_poems_arg="--poems-per-page $WORDCLOUD_POEMS"

2221 local chrono_per_page_arg="--chrono-per-page $CHRONO_PER_PAGE"

2222

2223 # The dry-run check comes BEFORE the clear below, and that ordering is a bug

2224 # fix, not a style choice: it used to come after, so --dry-run DELETED every

2225 # per-word page and then printed what it "would" do. Roughly 7,000 files and

2226 # 11 GB, removed by the one flag whose entire promise is that it changes

2227 # nothing. Any destructive step in a stage function belongs below this check.

2228 if $DRY_RUN; then

2229 log_dry_run "rm $OUTPUT_DIR/wordcloud/*.html (clear stale per-word pages)"

2230 log_dry_run "luajit $DIR/src/wordcloud-generator.lua $DIR $wordcloud_words_arg $chrono_per_page_arg $RANDOM_SEED_ARG $ASSETS_ARG"

2231 log_dry_run "luajit $DIR/src/generate-word-pages.lua $DIR --html-only $wordcloud_words_arg $wordcloud_poems_arg $chrono_per_page_arg $ASSETS_ARG"

2232 return 0

2233 fi

2234

2235 # Issue 10-059/10-061: wipe the per-word pages before regenerating. A word that

2236 # has fallen out of the cloud since the last build leaves an orphan page that the

2237 # generator never overwrites -- and an orphan from before a link-scheme change

2238 # ships BROKEN links (this is exactly how 134 stale "/similar-different/" pages

2239 # survived into a relative-path build). The pages are fully regenerated from the

2240 # current word set just below, so clearing every run (not only on --force) is

2241 # safe and is the only way to guarantee no stale orphans. Matches the principle

2242 # that each stage wipes its own output subdirectory before rebuilding it.

2243 if [ -d "$OUTPUT_DIR/wordcloud" ]; then

2244 log_info " Clearing stale per-word pages before regeneration..."

2245 rm -f "$OUTPUT_DIR/wordcloud/"*.html

2246 fi

2247

2248 # The word cloud IS the site's menu (and carries the live poem index), so a

2249 # failure here is fatal, not a warning -- there is no usable entry page without it.

2250 log_info " Generating word cloud menu..."

2251 $NICE_PREFIX luajit "$DIR/src/wordcloud-generator.lua" "$DIR" $wordcloud_words_arg $chrono_per_page_arg $RANDOM_SEED_ARG $ASSETS_ARG || {

2252 echo "Error: Word cloud menu generation failed" >&2

2253 exit 1

2254 }

2255

2256 log_info " Generating word similarity pages..."

2257 $NICE_PREFIX luajit "$DIR/src/generate-word-pages.lua" "$DIR" --html-only $wordcloud_words_arg $wordcloud_poems_arg $chrono_per_page_arg $ASSETS_ARG || {

2258 echo "Error: Word similarity page generation failed" >&2

2259 exit 1

2260 }

2261}

2262# }}}

2263

2264# }}}

2265

2266# {{{ interactive_mode_tui

2267# TUI-based interactive mode with command preview

2268# Uses Lua menu library for stable rendering and real-time command preview

2269interactive_mode_tui() {

2270 # Issue 10-065: both of these used to fall back to a different, older

2271 # interactive mode (luajit src/main.lua -I). That is the most consequential

2272 # fallback the script had, because it did not substitute a VALUE -- it

2273 # substituted a PROGRAM. The operator asked for the menu with the command

2274 # preview and the per-stage force checkboxes, and got a different interface

2275 # with a different set of options, after a message that scrolled past. The

2276 # older mode still exists and can be run directly; it is just not something

2277 # to be handed silently.

2278 if ! command -v tui_init >/dev/null; then

2279 echo "ERROR: the interactive menu library is not loaded." >&2

2280 echo " Expected: ${LIBS_DIR}/lua-menu.sh (and luajit on PATH)." >&2

2281 echo " -I needs it. Every non-interactive stage still works." >&2

2282 return 1

2283 fi

2284

2285 if ! tui_init; then

2286 echo "ERROR: the interactive menu failed to initialize." >&2

2287 echo " This usually means the terminal is not a TTY -- the menu" >&2

2288 echo " cannot render into a pipe or a captured stream." >&2

2289 return 1

2290 fi

2291

2292 # Build the menu

2293 menu_init

2294 menu_set_title "Neocities Pipeline" "Use j/k to navigate, space to toggle, Enter to run"

2295

2296 # ═══════════════════════════════════════════════════════════════════════════

2297 # Section 1: Pipeline Stages (multi - can select multiple)

2298 # Each checkbox maps to a CLI flag for command preview

2299 # Issue 10-016: Force regeneration moved here with per-stage options

2300 # ═══════════════════════════════════════════════════════════════════════════

2301 menu_add_section "stages" "multi" "Pipeline Stages (toggle stages to run)"

2302

2303 # Issue 10-016: Global force regenerate option at top of stages

2304 menu_add_item "stages" "force" "Force regenerate ALL stages" "checkbox" "0" \

2305 "Force regeneration even if files are fresh" "" "--force"

2306

2307 menu_add_item "stages" "update_words" "1. Update Words" "checkbox" "1" \

2308 "Sync input files from words repository" "" "--update-words"

2309 menu_add_item "stages" "force_update_words" " ↳ Force regenerate" "checkbox" "0" \

2310 "Force regenerate this stage only" "" "--force-stage 1"

2311

2312 menu_add_item "stages" "extract" "2. Extract" "checkbox" "1" \

2313 "Extract content from backup archives" "" "--extract"

2314 menu_add_item "stages" "force_extract" " ↳ Force regenerate" "checkbox" "0" \

2315 "Force regenerate this stage only" "" "--force-stage 2"

2316

2317 menu_add_item "stages" "parse" "3. Parse" "checkbox" "1" \

2318 "Parse poems from JSON sources into poems.json" "" "--parse"

2319 menu_add_item "stages" "force_parse" " ↳ Force regenerate" "checkbox" "0" \

2320 "Force regenerate this stage only" "" "--force-stage 3"

2321

2322 menu_add_item "stages" "validate" "4. Validate" "checkbox" "1" \

2323 "Run poem validation" "" "--validate"

2324 menu_add_item "stages" "force_validate" " ↳ Force regenerate" "checkbox" "0" \

2325 "Force regenerate this stage only" "" "--force-stage 4"

2326

2327 menu_add_item "stages" "catalog_images" "5. Catalog Images" "checkbox" "1" \

2328 "Catalog images from input directories" "" "--catalog-images"

2329 menu_add_item "stages" "force_catalog_images" " ↳ Force regenerate" "checkbox" "0" \

2330 "Force regenerate this stage only" "" "--force-stage 5"

2331

2332 menu_add_item "stages" "generate_embeddings" "6. Embeddings ⚠️" "checkbox" "0" \

2333 "Generate embeddings via the inference server" "" "--generate-embeddings"

2334 menu_add_item "stages" "force_generate_embeddings" " ↳ Force regenerate" "checkbox" "0" \

2335 "Force regenerate this stage only" "" "--force-stage 6"

2336

2337 menu_add_item "stages" "generate_similarity" "7. Similarity ⚠️" "checkbox" "0" \

2338 "Build similarity matrix" "" "--generate-similarity"

2339 menu_add_item "stages" "force_generate_similarity" " ↳ Force regenerate" "checkbox" "0" \

2340 "Force regenerate this stage only" "" "--force-stage 7"

2341

2342 menu_add_item "stages" "generate_diversity" "8. Diversity ⚠️" "checkbox" "0" \

2343 "Pre-compute diversity cache" "" "--generate-diversity"

2344 menu_add_item "stages" "force_generate_diversity" " ↳ Force regenerate" "checkbox" "0" \

2345 "Force regenerate this stage only" "" "--force-stage 8"

2346

2347 menu_add_item "stages" "generate_html" "9. Generate HTML" "checkbox" "1" \

2348 "Generate website HTML (chronological + similarity pages)" "" "--generate-html"

2349 menu_add_item "stages" "force_generate_html" " ↳ Force regenerate" "checkbox" "0" \

2350 "Force regenerate this stage only" "" "--force-stage 9"

2351

2352 menu_add_item "stages" "generate_wordcloud" "10. Generate Word Cloud" "checkbox" "1" \

2353 "Generate the word-cloud menu and per-word similarity pages" "" "--generate-wordcloud"

2354 menu_add_item "stages" "force_generate_wordcloud" " ↳ Force regenerate" "checkbox" "0" \

2355 "Force regenerate this stage only" "" "--force-stage 10"

2356

2357 # Issue 10-016: Dependencies - per-stage force options disabled when global force is checked

2358 # invert=true means: enable per-stage force when global force is NOT checked

2359 menu_add_dependency "force_update_words" "force" "1" "true" \

2360 "Disabled: global force is active" "orange"

2361 menu_add_dependency "force_extract" "force" "1" "true" \

2362 "Disabled: global force is active" "orange"

2363 menu_add_dependency "force_parse" "force" "1" "true" \

2364 "Disabled: global force is active" "orange"

2365 menu_add_dependency "force_validate" "force" "1" "true" \

2366 "Disabled: global force is active" "orange"

2367 menu_add_dependency "force_catalog_images" "force" "1" "true" \

2368 "Disabled: global force is active" "orange"

2369 menu_add_dependency "force_generate_embeddings" "force" "1" "true" \

2370 "Disabled: global force is active" "orange"

2371 menu_add_dependency "force_generate_similarity" "force" "1" "true" \

2372 "Disabled: global force is active" "orange"

2373 menu_add_dependency "force_generate_diversity" "force" "1" "true" \

2374 "Disabled: global force is active" "orange"

2375 menu_add_dependency "force_generate_html" "force" "1" "true" \

2376 "Disabled: global force is active" "orange"

2377 menu_add_dependency "force_generate_wordcloud" "force" "1" "true" \

2378 "Disabled: global force is active" "orange"

2379

2380 # ═══════════════════════════════════════════════════════════════════════════

2381 # Section 2: Configuration Options

2382 # ═══════════════════════════════════════════════════════════════════════════

2383 menu_add_section "config" "multi" "Configuration"

2384 # Issue 10-034: Orchestrator pattern enables parallel HTML with low memory

2385 # Main thread sends 80KB work slices instead of workers loading 700MB caches

2386 # Expected memory: ~2.5GB total (vs 14GB+ before fix)

2387 # Issue 10-065: the descriptions no longer advertise defaults, because there

2388 # are none. A blank field here is not "use the default" -- it is an absent

2389 # value, and it surfaces in the same missing-flags report a bare command line

2390 # produces. The numbers still shown in the field are the menu's pre-filled

2391 # SUGGESTIONS, which the operator can accept or overwrite; the distinction is

2392 # that a suggestion is visible on screen before it is used.

2393 menu_add_item "config" "threads" "Thread Count" "flag" "4:8" \

2394 "Threads for HTML gen (orchestrator mode)" "" "--threads"

2395 # Issue 8-022: Pagination options for HTML generation

2396 menu_add_item "config" "pages" "Pages per Poem" "flag" "1:2" \

2397 "Pages generated per poem" "" "--pages"

2398 menu_add_item "config" "poems_per_page" "Poems per Page" "flag" "200:3" \

2399 "Poems per similar/different page" "" "--poems-per-page"

2400 menu_add_item "config" "chrono_per_page" "Chrono per Page" "flag" "500:3" \

2401 "Poems per chronological page" "" "--chrono-per-page"

2402 # Issue 10-016: Force Regeneration moved to stages section

2403 menu_add_item "config" "dry_run" "Dry Run" "checkbox" "0" \

2404 "Show what would be executed without running" "" "--dry-run"

2405 menu_add_item "config" "verbose" "Verbose Output" "checkbox" "0" \

2406 "Show detailed progress information" "" "--verbose"

2407 # Issue 10-065: a checkbox always holds a definite state, so this one always

2408 # answers --boosts -- unchecked means "no", not "unanswered".

2409 menu_add_item "config" "include_boosts" "Include Boosts" "checkbox" "0" \

2410 "Include fediverse boosts/reblogs in extraction and parsing" "" "--boosts yes"

2411 # The seed stays a typed field because it is a number with no candidate list

2412 # to pick from -- any non-negative integer is equally valid, which is exactly

2413 # what makes it reproducible. Left EMPTY: pre-filling a seed would be the

2414 # auto-generated seed this issue removed, wearing a menu.

2415 menu_add_item "config" "seed" "Random Seed" "flag" ":12" \

2416 "Master seed for word-cloud shuffle and image order" "" "--seed"

2417

2418 # ═══════════════════════════════════════════════════════════════════════════

2419 # Section 2b: Model and server, as PICK-LISTS built from config.lua

2420 #

2421 # Issue 10-065: these are lists, not text boxes, and the reason is ergonomic

2422 # rather than technical. "qwen3-embedding:4b" is an exact string; typing it

2423 # from memory goes wrong eventually, and the wrong version is only discovered

2424 # when a stage fails. You cannot mistype something you pick.

2425 #

2426 # A "single" section renders as a radio group -- exactly one choice -- and

2427 # each entry carries the whole "--model NAME" as its CLI flag, so the command

2428 # preview shows the real, runnable text.

2429 #

2430 # The entries are read at menu-build time from config.lua via

2431 # scripts/list-inference-choices, so adding a server or a model to the config

2432 # makes it appear here with no edit to this file. That is the point: a

2433 # hand-maintained copy of the list in run.sh would be a second source of

2434 # truth, and second sources of truth are what this whole issue is about.

2435 _choice_servers="$("$DIR/scripts/list-inference-choices" "$DIR" --servers)" || {

2436 echo "ERROR: could not read the inference server list from config.lua" >&2

2437 return 1

2438 }

2439 _choice_models="$("$DIR/scripts/list-inference-choices" "$DIR" --models)" || {

2440 echo "ERROR: could not read the embedding model list from config.lua" >&2

2441 return 1

2442 }

2443

2444 # The names are kept in arrays alongside the menu items because the menu

2445 # library exposes menu_get_value but no menu_get_label -- so the checked item

2446 # can be identified by index, but its displayed name has to be remembered

2447 # here. Index i in the array is always item "model_i" / "server_i".

2448 MODEL_CHOICES=()

2449 SERVER_CHOICES=()

2450

2451 menu_add_section "model" "single" "Embedding Model (pick one)"

2452 while IFS= read -r _m; do

2453 [ -z "$_m" ] && continue

2454 # Item ids must be unique and shell-safe, so they are numbered rather

2455 # than derived from the model name (which contains ':' and '.').

2456 menu_add_item "model" "model_${#MODEL_CHOICES[@]}" "$_m" "checkbox" "0" \

2457 "Use the $_m embedding model" "" "--model $_m"

2458 MODEL_CHOICES+=("$_m")

2459 done <<< "$_choice_models"

2460

2461 menu_add_section "server" "single" "Inference Server (pick one)"

2462 while IFS= read -r _s; do

2463 [ -z "$_s" ] && continue

2464 menu_add_item "server" "server_${#SERVER_CHOICES[@]}" "$_s" "checkbox" "0" \

2465 "Send embedding requests to $_s" "" "--server $_s"

2466 SERVER_CHOICES+=("$_s")

2467 done <<< "$_choice_servers"

2468

2469 # ═══════════════════════════════════════════════════════════════════════════

2470 # Section 3: Word Cloud Configuration

2471 # Issue 8-043: Configurable word count with "all words" toggle

2472 # ═══════════════════════════════════════════════════════════════════════════

2473 menu_add_section "wordcloud" "multi" "Word Cloud Options"

2474 menu_add_item "wordcloud" "wordcloud_all" "All Words" "checkbox" "0" \

2475 "Include all words (disables word count limit)" "" "--wordcloud-words all"

2476 menu_add_item "wordcloud" "wordcloud_words" "Word Count" "flag" "200:3" \

2477 "Maximum words in word cloud" "" "--wordcloud-words"

2478 # Issue 8-050d: Poems per word-cloud page

2479 menu_add_item "wordcloud" "wordcloud_poems" "Poems Per Page" "flag" "50:3" \

2480 "Poems per word-cloud similarity page" "" "--wordcloud-poems"

2481 # Dependency: Disable wordcloud_words when wordcloud_all is checked

2482 # invert=true means: enable wordcloud_words when wordcloud_all is NOT checked (value "1")

2483 menu_add_dependency "wordcloud_words" "wordcloud_all" "1" "true" \

2484 "Word count disabled when 'All Words' is checked"

2485

2486 # ═══════════════════════════════════════════════════════════════════════════

2487 # Section 4: Command Preview (shows the command that will be executed)

2488 # ═══════════════════════════════════════════════════════════════════════════

2489 menu_add_section "preview" "multi" "Command Preview"

2490 menu_add_item "preview" "cmd_preview" "" "text" "" \

2491 "The command that will be executed (press ~ to copy to clipboard)"

2492

2493 # Configure command preview - links checkboxes to command string

2494 menu_set_command_config "./run.sh" "cmd_preview" ""

2495

2496 # ═══════════════════════════════════════════════════════════════════════════

2497 # Section 5: Actions

2498 # ═══════════════════════════════════════════════════════════════════════════

2499 menu_add_section "actions" "single" "Actions"

2500 menu_add_item "actions" "run" "Run Selected Stages" "action" "" \

2501 "Execute the selected pipeline stages" ""

2502

2503 # Run the menu loop

2504 while true; do

2505 if menu_run; then

2506 # User selected "run" - extract values and execute

2507 local update_words_val=$(menu_get_value "update_words")

2508 local extract_val=$(menu_get_value "extract")

2509 local parse_val=$(menu_get_value "parse")

2510 local validate_val=$(menu_get_value "validate")

2511 local catalog_val=$(menu_get_value "catalog_images")

2512 local embeddings_val=$(menu_get_value "generate_embeddings")

2513 local similarity_val=$(menu_get_value "generate_similarity")

2514 local diversity_val=$(menu_get_value "generate_diversity")

2515 local html_val=$(menu_get_value "generate_html")

2516 local wordcloud_stage_val=$(menu_get_value "generate_wordcloud")

2517 local threads_val=$(menu_get_value "threads")

2518 # Issue 8-022: Get pagination values from TUI

2519 local pages_val=$(menu_get_value "pages")

2520 local poems_per_page_val=$(menu_get_value "poems_per_page")

2521 local chrono_per_page_val=$(menu_get_value "chrono_per_page")

2522 local force_val=$(menu_get_value "force")

2523 # Issue 10-016: Get per-stage force values from TUI

2524 local force_update_words_val=$(menu_get_value "force_update_words")

2525 local force_extract_val=$(menu_get_value "force_extract")

2526 local force_parse_val=$(menu_get_value "force_parse")

2527 local force_validate_val=$(menu_get_value "force_validate")

2528 local force_catalog_val=$(menu_get_value "force_catalog_images")

2529 local force_embeddings_val=$(menu_get_value "force_generate_embeddings")

2530 local force_similarity_val=$(menu_get_value "force_generate_similarity")

2531 local force_diversity_val=$(menu_get_value "force_generate_diversity")

2532 local force_html_val=$(menu_get_value "force_generate_html")

2533 local force_wordcloud_val=$(menu_get_value "force_generate_wordcloud")

2534 local dry_val=$(menu_get_value "dry_run")

2535 local verbose_val=$(menu_get_value "verbose")

2536 # Issue 8-011: Get boost inclusion value from TUI

2537 local include_boosts_val=$(menu_get_value "include_boosts")

2538 # Issue 8-043: Get wordcloud values from TUI

2539 local wordcloud_all_val=$(menu_get_value "wordcloud_all")

2540 local wordcloud_words_val=$(menu_get_value "wordcloud_words")

2541 # Issue 8-050d: Get poems per word-cloud page from TUI

2542 local wordcloud_poems_val=$(menu_get_value "wordcloud_poems")

2543 # Issue 10-065: the seed, and the two pick-lists.

2544 local seed_val=$(menu_get_value "seed")

2545

2546 # Set global flags based on menu selection

2547 [[ "$update_words_val" == "1" ]] && UPDATE_WORDS=true || UPDATE_WORDS=false

2548 [[ "$extract_val" == "1" ]] && EXTRACT=true || EXTRACT=false

2549 [[ "$parse_val" == "1" ]] && PARSE=true || PARSE=false

2550 [[ "$validate_val" == "1" ]] && VALIDATE=true || VALIDATE=false

2551 [[ "$catalog_val" == "1" ]] && CATALOG_IMAGES=true || CATALOG_IMAGES=false

2552 [[ "$embeddings_val" == "1" ]] && GENERATE_EMBEDDINGS=true || GENERATE_EMBEDDINGS=false

2553 [[ "$similarity_val" == "1" ]] && GENERATE_SIMILARITY=true || GENERATE_SIMILARITY=false

2554 [[ "$diversity_val" == "1" ]] && GENERATE_DIVERSITY=true || GENERATE_DIVERSITY=false

2555 [[ "$html_val" == "1" ]] && GENERATE_HTML=true || GENERATE_HTML=false

2556 [[ "$wordcloud_stage_val" == "1" ]] && GENERATE_WORDCLOUD=true || GENERATE_WORDCLOUD=false

2557

2558 # Config flags. Issue 10-065: a blank menu field leaves the variable

2559 # empty, and empty is exactly what the requirement gate reports as

2560 # missing -- the menu and the command line reach the same gate, so

2561 # neither route can start a build with a value nobody chose.

2562 [[ -n "$threads_val" && "$threads_val" != "0" ]] && THREADS="$threads_val"

2563 # Issue 8-022: Set pagination values from TUI

2564 [[ -n "$pages_val" && "$pages_val" != "0" ]] && PAGES="$pages_val"

2565 [[ -n "$poems_per_page_val" && "$poems_per_page_val" != "0" ]] && POEMS_PER_PAGE="$poems_per_page_val"

2566 [[ -n "$chrono_per_page_val" && "$chrono_per_page_val" != "0" ]] && CHRONO_PER_PAGE="$chrono_per_page_val"

2567 [[ "$force_val" == "1" ]] && FORCE=true || FORCE=false

2568 # Issue 10-016: Set per-stage force flags from TUI

2569 [[ "$force_update_words_val" == "1" ]] && FORCE_STAGE_1=true || FORCE_STAGE_1=false

2570 [[ "$force_extract_val" == "1" ]] && FORCE_STAGE_2=true || FORCE_STAGE_2=false

2571 [[ "$force_parse_val" == "1" ]] && FORCE_STAGE_3=true || FORCE_STAGE_3=false

2572 [[ "$force_validate_val" == "1" ]] && FORCE_STAGE_4=true || FORCE_STAGE_4=false

2573 [[ "$force_catalog_val" == "1" ]] && FORCE_STAGE_5=true || FORCE_STAGE_5=false

2574 [[ "$force_embeddings_val" == "1" ]] && FORCE_STAGE_6=true || FORCE_STAGE_6=false

2575 [[ "$force_similarity_val" == "1" ]] && FORCE_STAGE_7=true || FORCE_STAGE_7=false

2576 [[ "$force_diversity_val" == "1" ]] && FORCE_STAGE_8=true || FORCE_STAGE_8=false

2577 [[ "$force_html_val" == "1" ]] && FORCE_STAGE_9=true || FORCE_STAGE_9=false

2578 [[ "$force_wordcloud_val" == "1" ]] && FORCE_STAGE_10=true || FORCE_STAGE_10=false

2579 [[ "$dry_val" == "1" ]] && DRY_RUN=true || DRY_RUN=false

2580 [[ "$verbose_val" == "1" ]] && VERBOSE=true || VERBOSE=false

2581 # Issue 8-011/10-065: the boost checkbox now answers --boosts with a

2582 # definite yes or no. A checkbox cannot be "unanswered", which is why

2583 # this is the one required value the menu can always supply.

2584 if [[ "$include_boosts_val" == "1" ]]; then

2585 BOOSTS="yes"

2586 else

2587 BOOSTS="no"

2588 fi

2589 # Issue 8-043: Set the word count from the TUI. The "All Words" checkbox

2590 # wins -- it sets the count to the literal "all" (and the dependency has

2591 # already disabled the now-irrelevant Word Count field). Otherwise the

2592 # typed count is used. One value, WORDCLOUD_WORDS, feeds --wordcloud-words.

2593 if [[ "$wordcloud_all_val" == "1" ]]; then

2594 WORDCLOUD_WORDS="all"

2595 elif [[ -n "$wordcloud_words_val" && "$wordcloud_words_val" != "0" ]]; then

2596 WORDCLOUD_WORDS="$wordcloud_words_val"

2597 fi

2598 # Issue 8-050d: Set poems per word-cloud page from TUI

2599 [[ -n "$wordcloud_poems_val" && "$wordcloud_poems_val" != "0" ]] && WORDCLOUD_POEMS="$wordcloud_poems_val"

2600

2601 # Issue 10-065: the seed. No "!= 0" guard here, unlike the fields

2602 # above: zero is a PERFECTLY VALID seed, and treating it as "unset"

2603 # would silently reject a reproducible build the operator asked for.

2604 # (The guard exists on the others because the menu uses "0" to mean

2605 # an untouched numeric field, and zero threads or zero pages are

2606 # meaningless anyway.)

2607 [[ -n "$seed_val" ]] && RANDOM_SEED="$seed_val"

2608

2609 # The model and server pick-lists. Each entry is a checkbox in a

2610 # "single" section, so at most one is checked; walk them and take the

2611 # one that is, reading its name out of the array built alongside the

2612 # menu items (index i <-> item "model_i").

2613 #

2614 # Nothing is assigned when nothing is checked: the variable stays

2615 # empty and the requirement gate reports it as missing, which is the

2616 # same outcome as omitting the flag on the command line. Both doors,

2617 # one gate.

2618 local _i _val

2619 _i=0

2620 while [ "$_i" -lt "${#MODEL_CHOICES[@]}" ]; do

2621 _val=$(menu_get_value "model_$_i")

2622 [[ "$_val" == "1" ]] && MODEL_NAME="${MODEL_CHOICES[$_i]}"

2623 _i=$((_i + 1))

2624 done

2625 _i=0

2626 while [ "$_i" -lt "${#SERVER_CHOICES[@]}" ]; do

2627 _val=$(menu_get_value "server_$_i")

2628 [[ "$_val" == "1" ]] && INFERENCE_SERVER="${SERVER_CHOICES[$_i]}"

2629 _i=$((_i + 1))

2630 done

2631

2632 # Check if at least one stage is selected

2633 if ! $UPDATE_WORDS && ! $EXTRACT && ! $PARSE && ! $VALIDATE && \

2634 ! $CATALOG_IMAGES && ! $GENERATE_EMBEDDINGS && ! $GENERATE_SIMILARITY && \

2635 ! $GENERATE_DIVERSITY && ! $GENERATE_HTML && ! $GENERATE_WORDCLOUD; then

2636 echo ""

2637 echo "No stages selected. Please select at least one stage to run."

2638 echo "Press Enter to continue..."

2639 read -r

2640 continue

2641 fi

2642

2643 # Exit menu and run the pipeline

2644 menu_cleanup

2645 return 0

2646 else

2647 # User quit

2648 menu_cleanup

2649 echo "Goodbye!"

2650 exit 0

2651 fi

2652 done

2653}

2654# }}}

2655

2656# {{{ Main execution

2657

2658# Handle interactive mode

2659EXECUTED_COMMAND="" # Store command for post-run display

2660if $INTERACTIVE; then

2661 log_info "🎛️ Launching interactive mode with command preview..."

2662 interactive_mode_tui || {

2663 echo "Error: interactive mode could not start." >&2

2664 exit 1

2665 }

2666 # Save the command preview for display after execution

2667 EXECUTED_COMMAND=$(menu_get_value "cmd_preview")

2668 # After TUI, fall through to execute selected stages

2669fi

2670

2671# {{{ The requirement gate (Issue 10-065)

2672# Placed HERE, and the position is the design. Two things must already have

2673# happened and one must not have:

2674#

2675# The stage selection must be final -- requirements follow the selected

2676# stages, and the menu above is the second way stages get selected.

2677# The menu's values must be in, for the same reason: menu and command line

2678# are two doors into one gate, and a build must not be startable through

2679# either one with a value nobody chose.

2680# Nothing may have RESOLVED a value yet. The model resolution below used to

2681# sit hundreds of lines above this point, where it read config.lua whenever

2682# --model was absent. A resolver that runs before the gate makes the gate

2683# decorative: it would find the value present, because the resolver had just

2684# invented it.

2685collect_missing_values

2686if [ "${#MISSING_VALUES[@]}" -gt 0 ] && $INTERACTIVE; then

2687 # Every required value has somewhere to come from in the menu, so an absence

2688 # here means a field was left blank or a list left unpicked -- not that the

2689 # menu was incapable of expressing it. Point at the section rather than at

2690 # the command line.

2691 echo "" >&2

2692 echo "Note: the menu can supply all of these -- the Configuration section" >&2

2693 echo " for the numbers and the seed, and the Embedding Model and" >&2

2694 echo " Inference Server pick-lists for the names." >&2

2695fi

2696report_missing_values

2697validate_supplied_values

2698# }}}

2699

2700# {{{ Record this run's choices where the child programs will find them

2701# Why this exists: run.sh launches a fresh luajit process per stage, and argv

2702# reaches only the stages we remember to thread it through. Before the notepad, a

2703# --model override silently reverted to config.lua's default in the HTML,

2704# word-cloud and word-page stages (they resolve the model via get_selected_model()

2705# / embeddings_dir() with no argument). The fix is a shared notepad in RAM: this

2706# run's choices are stamped onto tmp/shared-memory/run-overrides.lua once, here,

2707# and the resolver reads them. It is rewritten every run, so a previous run's

2708# choice can never leak in -- the staleness trap a file has but an env var does

2709# not.

2710#

2711# Issue 10-065: the server is recorded alongside the model, and BOTH are recorded

2712# only when this run actually has them. Which stages require a model is decided by

2713# the requirements table above, and stages 1-5 do not -- so a --validate run

2714# reaches this point with MODEL_NAME empty, legitimately. The notepad writer

2715# already skips empty values (an absent key means "no override"), so passing them

2716# through unguarded is safe here; the cache-directory block below is where an

2717# empty model was NOT safe. See its comment.

2718"$DIR/scripts/write-run-overrides" "$DIR" \

2719 --model "$MODEL_NAME" \

2720 --server "$INFERENCE_SERVER" || {

2721 echo "Error: failed to record run overrides (scripts/write-run-overrides)" >&2

2722 exit 1

2723}

2724

2725# Create this model's cache directories ONCE here, instead of making each stage

2726# remember to mkdir its own output dir before its first write. The paths are

2727# inferred from the model name by scripts/cache-dir (the single place that maps a

2728# model -> its directories): the movable (RAM) dir, its similarities/ subdir, and

2729# the reboot-surviving on-disk dir (--disk). A brand-new model otherwise has no

2730# assets/embeddings/<model>/ folder, which once let a 40-minute diversity run

2731# finish and then fail at its final write. Adding a new model now needs no manual

2732# mkdir -- selecting it is enough.

2733#

2734# Issue 10-065: guarded on having a model at all. This block used to run on EVERY

2735# invocation, and `scripts/cache-dir --model ""` resolves to the embeddings ROOT

2736# rather than to a model's subdirectory -- so `./run.sh --validate`, a stage that

2737# requires no model, created a stray `cache/embeddings/similarities/` one level

2738# above where any similarity file belongs. Found by noticing that exact directory

2739# on this machine and matching its timestamp to a --validate --dry-run.

2740if [ -n "$MODEL_NAME" ]; then

2741 _ram_dir="$(luajit "$DIR/scripts/cache-dir" "$DIR" --model "$MODEL_NAME")"

2742 _disk_dir="$(luajit "$DIR/scripts/cache-dir" "$DIR" --model "$MODEL_NAME" --disk)"

2743 if [ -z "$_ram_dir" ] || [ -z "$_disk_dir" ]; then

2744 echo "Error: could not resolve cache directories for model $MODEL_NAME" >&2

2745 exit 1

2746 fi

2747 # The dry-run guard is the same rule the stage functions follow: --dry-run

2748 # must not change anything. This mkdir used to run regardless, so a --dry-run

2749 # with a mistyped --model left a real, empty cache directory behind -- named

2750 # after a model that does not exist, sitting beside the real ones, and

2751 # indistinguishable from a model whose embeddings simply have not been

2752 # generated yet.

2753 if $DRY_RUN; then

2754 log_dry_run "mkdir -p $_ram_dir/similarities $_disk_dir (cache dirs for $MODEL_NAME)"

2755 else

2756 mkdir -p "$_ram_dir/similarities" "$_disk_dir" || {

2757 echo "Error: could not create cache directories for model $MODEL_NAME" >&2

2758 exit 1

2759 }

2760 fi

2761fi

2762# }}}

2763

2764# {{{ The build's master seed

2765# One integer governs every randomization site this run (the word-cloud shuffle

2766# and image-order randomization).

2767#

2768# Resolution: --seed if given, otherwise a fresh random one. An unseeded build is

2769# NOT refused, because refusing would not make anyone's choice more deliberate --

2770# it would just make them type a number to get past a prompt. What makes an

2771# unseeded build reproducible is that the seed is RECORDED, which is Issue

2772# 10-058's design and is what happens below: it is logged, written into

2773# generation-metadata.json, and stamped into the word-cloud page itself, so any

2774# archived cloud can be re-created from the file.

2775#

2776# Entropy: the epoch second alone would give two runs in the same second the same

2777# seed, so the process id is mixed in. Folded to a 31-bit non-negative integer so

2778# it round-trips unchanged through a command line, JSON, and math.randomseed.

2779if [ -z "$RANDOM_SEED" ]; then

2780 RANDOM_SEED=$(( ($(date +%s) * 100000 + $$) % 2147483647 ))

2781 RANDOM_SEED_SOURCE="randomized (no --seed given)"

2782else

2783 RANDOM_SEED_SOURCE="--seed"

2784fi

2785

2786# The argument every randomizing subprocess receives. Equals-form on purpose: the

2787# bare number can never be mistaken for a positional DIR by a child's arg parser.

2788RANDOM_SEED_ARG="--seed=$RANDOM_SEED"

2789log_info "🎲 Random seed: $RANDOM_SEED ($RANDOM_SEED_SOURCE)"

2790# }}}

2791

2792# {{{ Record what produced this build

2793# Written when this run required at least one value -- which is the same as

2794# saying it did something whose parameters are worth recording. Stages 1-4

2795# require nothing, so a --validate run writes no record and, importantly, does

2796# not OVERWRITE the existing one: that file still truthfully describes the build

2797# whose pages are sitting in output/ right now, which validating did not touch.

2798# Erasing it would be the fallback pattern in reverse -- replacing a value

2799# somebody supplied with one nobody did.

2800#

2801# Known limitation, worth stating rather than hiding: the file is replaced, not

2802# merged. output/ can hold pages from several runs (stage 9 today, stage 10 last

2803# week), and a replaced record then describes only the most recent. Merging would

2804# be more truthful and needs a JSON reader this shell script does not have.

2805_run_required_something=false

2806for _row in "${REQUIRED_VALUES[@]}"; do

2807 IFS=';' read -r _var _usage _reason _consumers _key <<< "$_row"

2808 IFS=',' read -r -a _entries <<< "$_consumers"

2809 for _entry in "${_entries[@]}"; do

2810 _sv="${_entry%%:*}"

2811 [ "${!_sv}" = "true" ] && _run_required_something=true

2812 done

2813done

2814# The seed left the requirements table, so the stages that consume it have to be

2815# named here or a run of stage 5 alone (image-order randomization, no other

2816# required values) would randomize the gallery and record nothing about it.

2817$CATALOG_IMAGES && _run_required_something=true

2818$GENERATE_WORDCLOUD && _run_required_something=true

2819

2820if $_run_required_something; then

2821 if $DRY_RUN; then

2822 log_dry_run "write $OUTPUT_DIR/generation-metadata.json (this run's values)"

2823 else

2824 write_generation_metadata

2825 fi

2826fi

2827# }}}

2828

2829# Show what will be executed (in non-interactive or after TUI selection)

2830if $DRY_RUN || $VERBOSE; then

2831 echo "Pipeline stages to execute:"

2832 # Issue 10-051 / alignment: render the plan as a TABLE -- stage names in one

2833 # left-aligned column, the measured average time right-aligned in the next --

2834 # so durations line up and the eye can scan them. Measured wall-clock (avg of

2835 # recent runs) appears once a stage has run here before; until then a coarse

2836 # magnitude word (short/medium/long) stands in, since a word can't go stale

2837 # the way a hard number can. The ⚠ marks the heavy stages.

2838 #

2840 # key can differ from the display name (word-cloud history is stored under

2841 # "wordcloud" but shown as "generate-wordcloud").

2842 _plan_rows=(

2853 )

2854 # Issue 10-065: the timing library is required to load (see "Setup

2855 # directories"), so this asks whether it exports the reader function, not

2856 # whether it is present. If it loaded but is missing the function, that is a

2857 # broken library rather than an absent one, and saying so beats printing a

2858 # plan with silently degraded estimates.

2859 if ! command -v stage_timing_mean >/dev/null; then

2860 echo "Error: scripts/stage-timing.sh loaded but exports no stage_timing_mean." >&2

2861 exit 1

2862 fi

2863

2864 # Pass 1: collect enabled rows + each one's time string and tail, and track

2865 # the widest label and widest time. The ⚠ glyph is counted as ONE display

2866 # column (not its byte length) so the multibyte char does not skew alignment.

2867 _p_num=(); _p_label=(); _p_lvis=(); _p_time=(); _p_tail=()

2868 _labelw=0; _timew=0

2869 for _row in "${_plan_rows[@]}"; do

2870 IFS='|' read -r _en _num _name _warn _key _mag <<< "$_row"

2871 [ "$_en" = "true" ] || continue

2872 _lbl="$_name"; _lvis=${#_name}

2873 if [ "$_warn" = "1" ]; then _lbl="$_name $(symbol_warning "⚠")"; _lvis=$(( ${#_name} + 2 )); fi

2874 _time=""; _tail="$_mag"

2875 # An empty mean is legitimate here and is NOT a fallback: it means this

2876 # stage has no recorded history yet, so the coarse magnitude word stands

2877 # in until it does. The distinction from a fallback is that the estimate

2878 # is labelled -- "(medium)" versus "(avg 4m 12s, last 3 runs)" -- so the

2879 # reader can always tell a measurement from a guess.

2880 _mean="$(stage_timing_mean "$_key")"

2881 if [ -n "$_mean" ]; then

2882 _cnt="$(stage_timing_count "$_key")"

2883 _pl="s"; [ "$_cnt" = "1" ] && _pl=""

2884 _time="$(stage_timing_format_seconds "$_mean")"

2885 _tail="last ${_cnt} run${_pl}"

2886 fi

2887 _p_num+=("$_num"); _p_label+=("$_lbl"); _p_lvis+=("$_lvis")

2888 _p_time+=("$_time"); _p_tail+=("$_tail")

2889 [ "$_lvis" -gt "$_labelw" ] && _labelw=$_lvis

2890 [ "${#_time}" -gt "$_timew" ] && _timew=${#_time}

2891 done

2892

2893 # Pass 2: print aligned. Number in a 3-wide field ("1." / "10."), label padded

2894 # to _labelw, time right-aligned to _timew inside "(avg <time>, <tail>)".

2895 _i=0

2896 while [ "$_i" -lt "${#_p_num[@]}" ]; do

2897 _pad=$(( _labelw - ${_p_lvis[$_i]} ))

2898 _sp=""; [ "$_pad" -gt 0 ] && _sp="$(printf '%*s' "$_pad" '')"

2899 if [ -n "${_p_time[$_i]}" ]; then

2900 printf " %-3s %s%s (avg %*s, %s)\n" \

2901 "${_p_num[$_i]}." "${_p_label[$_i]}" "$_sp" \

2902 "$_timew" "${_p_time[$_i]}" "${_p_tail[$_i]}"

2903 else

2904 printf " %-3s %s%s (%s)\n" \

2905 "${_p_num[$_i]}." "${_p_label[$_i]}" "$_sp" "${_p_tail[$_i]}"

2906 fi

2907 _i=$(( _i + 1 ))

2908 done

2909 echo ""

2910fi

2911

2912# {{{ Issue 10-017: Validate Inference server connectivity before embedding stages

2913if $GENERATE_EMBEDDINGS && ! $DRY_RUN; then

2914 log_info "Validating Inference server connectivity..."

2915 VALIDATION_RESULT=$(luajit -e "

2916 package.path = '$DIR/libs/?.lua;' .. package.path

2917 local inference = require('inference-server-config')

2918 -- Issue 10-065: unconditional. --server is required by this stage, so

2919 -- there is no empty-means-let-config-decide case -- which matters here

2920 -- more than anywhere, because this block decides which endpoint to

2921 -- health-check and, failing that, which one to START.

2922 inference.set_selected_server('$INFERENCE_SERVER')

2923 local server = inference.get_selected_server()

2924 local ok, msg = inference.validate_server(server)

2925 if ok then

2926 print('OK:' .. server.name .. ':' .. inference.build_host_url(server))

2927 else

2928 print('FAIL:' .. server.name .. ':' .. msg)

2929 end

2930 " 2>&1)

2931

2932 if [[ "$VALIDATION_RESULT" == OK:* ]]; then

2933 SERVER_NAME=$(echo "$VALIDATION_RESULT" | cut -d: -f2)

2934 SERVER_URL=$(echo "$VALIDATION_RESULT" | cut -d: -f3-)

2935 log_info " ✓ Inference server '$SERVER_NAME' is reachable at $SERVER_URL"

2936 else

2937 # Server unreachable. Try to start it ourselves (and remember we

2938 # did, so the EXIT trap shuts it down again). If start succeeds,

2939 # re-validate to confirm /health is responsive before proceeding.

2940 SERVER_NAME=$(echo "$VALIDATION_RESULT" | cut -d: -f2)

2941 ERROR_MSG=$(echo "$VALIDATION_RESULT" | cut -d: -f3-)

2942 log_info " ✗ Inference server '$SERVER_NAME' not reachable: $ERROR_MSG"

2943 log_info " Attempting to start it via scripts/start-llamacpp-server.sh..."

2944

2945 # Issue 10-065: the server name is always passed. It used to be appended

2946 # only when non-empty, so an omitted --server meant this script started

2947 # whichever server config.lua named -- and then the EXIT trap shut down

2948 # a server the operator had never mentioned.

2949 START_ARGS=("$DIR" "--server=$INFERENCE_SERVER")

2950 if "$DIR/scripts/start-llamacpp-server.sh" "${START_ARGS[@]}"; then

2951 WE_STARTED_INFERENCE_SERVER=true

2952

2953 # Re-validate to confirm the freshly-started server is responsive.

2954 VALIDATION_RESULT=$(luajit -e "

2955 package.path = '$DIR/libs/?.lua;' .. package.path

2956 local inference = require('inference-server-config')

2957 -- Issue 10-065: unconditional, matching the first validation

2958 -- block above. --server is required by this stage, so the

2959 -- empty case cannot arise.

2960 inference.set_selected_server('$INFERENCE_SERVER')

2961 local server = inference.get_selected_server()

2962 local ok, msg = inference.validate_server(server)

2963 if ok then

2964 print('OK:' .. server.name .. ':' .. inference.build_host_url(server))

2965 else

2966 print('FAIL:' .. server.name .. ':' .. msg)

2967 end

2968 " 2>&1)

2969

2970 if [[ "$VALIDATION_RESULT" == OK:* ]]; then

2971 SERVER_NAME=$(echo "$VALIDATION_RESULT" | cut -d: -f2)

2972 SERVER_URL=$(echo "$VALIDATION_RESULT" | cut -d: -f3-)

2973 log_info " ✓ Inference server '$SERVER_NAME' started at $SERVER_URL"

2974 log_info " (will be shut down again when this run completes)"

2975 else

2976 ERROR_MSG=$(echo "$VALIDATION_RESULT" | cut -d: -f3-)

2977 echo -e "${RED}❌ ERROR: Started the inference server but it is still not reachable${NC}" >&2

2978 echo -e "${RED} $ERROR_MSG${NC}" >&2

2979 # Issue 10-065: the log directory is stated rather than guessed

2980 # at with a ":-" default. --debug moves these logs to durable

2981 # disk; without it they are in the RAM-backed tmp/ and a reboot

2982 # takes them, which is precisely what an operator chasing a

2983 # server that will not start needs to be told.

2984 if [ -n "$NEOCITIES_LOG_DIR" ]; then

2985 echo -e "${YELLOW}💡 Check $NEOCITIES_LOG_DIR/llamacpp-server.log for the server's own diagnostics${NC}" >&2

2986 else

2987 echo -e "${YELLOW}💡 Check $DIR/tmp/llamacpp-server.log for the server's own diagnostics${NC}" >&2

2988 echo -e "${YELLOW} (that is RAM-backed and a reboot wipes it; re-run with --debug to keep it)${NC}" >&2

2989 fi

2990 exit 1

2991 fi

2992 else

2993 echo -e "${RED}❌ ERROR: Failed to start the inference server${NC}" >&2

2994 echo -e "${YELLOW}💡 Run ./scripts/start-llamacpp-server.sh manually for verbose output${NC}" >&2

2995 echo -e "${YELLOW}💡 Use --list-servers to see available servers${NC}" >&2

2996 echo -e "${YELLOW}💡 Use --server=NAME to select a different server${NC}" >&2

2997 exit 1

2998 fi

2999 fi

3000fi

3001# }}}

3002

3003# Execute stages in pipeline order (regardless of argument order)

3004# Issue 10-051: timed_stage <name> wraps each stage so its wall-clock is recorded

3005# to .stage-timings on success (skipped stages and failures record nothing). The

3006# names here are the keys the pre-flight list reads back for its estimates.

3007$UPDATE_WORDS && timed_stage update-words run_update_words

3008$EXTRACT && timed_stage extract run_extract

3009# Issue 10-053: strip excluded content from input/ right after sync/extraction,

3010# before anything catalogs or embeds it. Tied to extraction (which follows sync).

3011$EXTRACT && timed_stage strip-excluded run_strip_excluded

3012$PARSE && timed_stage parse run_parse

3013$VALIDATE && timed_stage validate run_validate

3014$CATALOG_IMAGES && timed_stage catalog-images run_catalog_images

3015$GENERATE_EMBEDDINGS && timed_stage generate-embeddings run_generate_embeddings

3016# Semantic colors are part of embedding generation (Stage 6.5)

3017# Only regenerate when embeddings are generated - HTML should use existing poem_colors.json

3018$GENERATE_EMBEDDINGS && timed_stage generate-semantic-colors run_generate_semantic_colors

3019# Word embeddings run AFTER colors so the word-color step finds color_embeddings.json

3020$GENERATE_EMBEDDINGS && timed_stage generate-word-embeddings run_generate_word_embeddings

3021# Issue 9-013: fold image pseudo-embeddings into the set BEFORE the similarity

3022# matrix is built, so images rank alongside poems. Idempotent + cheap.

3023$GENERATE_SIMILARITY && timed_stage augment-images run_augment_images

3024$GENERATE_SIMILARITY && timed_stage generate-similarity run_generate_similarity

3025$GENERATE_DIVERSITY && timed_stage generate-diversity run_generate_diversity

3026$GENERATE_HTML && timed_stage generate-html run_generate_html

3027$GENERATE_WORDCLOUD && timed_stage wordcloud run_generate_wordcloud

3028

3029if ! $QUIET; then

3030 echo ""

3031 echo -e "$(symbol_success "✅") Pipeline completed successfully"

3032

3033 # Print the executed command for easy re-running (copy-paste friendly)

3034 if [[ -n "$EXECUTED_COMMAND" ]]; then

3035 echo ""

3036 echo -e "$(symbol_info "📋") Command executed:"

3037 echo " $EXECUTED_COMMAND"

3038 fi

3039fi

3040# }}}

3041