run.sh

868resolve_random_seed() {

869 if [ -n "$RANDOM_SEED_FLAG" ]; then

870 RANDOM_SEED="$RANDOM_SEED_FLAG"

871 RANDOM_SEED_SOURCE="cli (--seed)"

872 return

873 fi

874 # config.lua is a static `return {...}` table, so dofile reads it without the

875 # Lua config-loader. pcall guards a malformed config (empty => fall through).

876 local cfg_seed

877 cfg_seed=$(luajit -e 'local ok,c=pcall(dofile,"'"$DIR"'/config.lua"); if ok and type(c)=="table" and c.randomization and c.randomization.seed then io.write(tostring(c.randomization.seed)) end')

878 if [ -n "$cfg_seed" ]; then

879 RANDOM_SEED="$cfg_seed"

880 RANDOM_SEED_SOURCE="config.lua (randomization.seed)"

881 return

882 fi

883 # Auto: mix epoch seconds with the PID so two runs in the same second differ;

884 # fold to a 31-bit non-negative int so it round-trips through CLI/JSON/randomseed.

885 RANDOM_SEED=$(( ($(date +%s) * 100000 + $$) % 2147483647 ))

886 RANDOM_SEED_SOURCE="auto-generated"

887}

911write_generation_metadata() {

912 local out_dir="${OUTPUT_DIR:-$DIR/output}"

913 mkdir -p "$out_dir"

914 local generated_at

915 generated_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)

916 cat > "$out_dir/generation-metadata.json" <<EOF

917{

918 "seed": $RANDOM_SEED,

919 "seed_source": "$RANDOM_SEED_SOURCE",

920 "generated_at": "$generated_at",

921 "pages": "${PAGES:-default}",

922 "poems_per_page": "${POEMS_PER_PAGE:-default}"

923}

924EOF

925}

938run_update_words() {

939 log_stage "📁 Stage 1/10: Updating input files from words repository"

941 # Issue 10-016: Check both global and per-stage force flags (Stage 1)

942 local stage_force=$FORCE

943 $FORCE_STAGE_1 && stage_force=true

945 # Issue 7-003: Pass force flag to skip file preservation

946 local force_flag=""

947 if $stage_force; then

948 force_flag="--force"

949 fi

951 if $DRY_RUN; then

952 log_dry_run "$DIR/scripts/update-words $force_flag"

953 return 0

954 fi

956 "$DIR/scripts/update-words" $force_flag || {

957 echo "Warning: Failed to update input files, continuing anyway..." >&2

958 }

959}

963run_extract() {

964 log_stage "🔄 Stage 2/10: Extracting content from backup archives"

966 # Issue 8-011: Build boost inclusion flag

967 local boost_flag=""

968 if $INCLUDE_BOOSTS; then

969 boost_flag="--include-boosts"

970 fi

972 if $DRY_RUN; then

973 log_dry_run "$DIR/scripts/update $DIR $boost_flag"

974 return 0

975 fi

977 "$DIR/scripts/update" "$DIR" $boost_flag || {

978 echo "Error: Content extraction failed" >&2

979 exit 1

980 }

981}

992run_strip_excluded() {

993 log_stage "🧹 Stripping excluded content from input/"

994 if $DRY_RUN; then

995 log_dry_run "lua $DIR/scripts/strip-excluded $DIR"

996 return 0

997 fi

998 if ! lua "$DIR/scripts/strip-excluded" "$DIR"; then

999 echo "ERROR: strip-excluded failed -- a broken exclusion path in config.lua." >&2

1000 echo " Fix excluded_images and re-run; nothing was stripped or shipped." >&2

1001 exit 1

1002 fi

1003}

1007run_parse() {

1008 log_stage "📝 Stage 3/10: Parsing poems from JSON sources"

1010 # Issue 10-016: Check both global and per-stage force flags (Stage 3)

1011 local stage_force=$FORCE

1012 $FORCE_STAGE_3 && stage_force=true

1014 local force_arg=""

1015 if $stage_force; then

1016 force_arg="--force"

1017 fi

1019 if $DRY_RUN; then

1020 log_dry_run "luajit src/main.lua $DIR --parse-only $force_arg $BOOSTS_ARG $ASSETS_ARG"

1021 return 0

1022 fi

1024 luajit src/main.lua "$DIR" --parse-only $force_arg $BOOSTS_ARG $ASSETS_ARG || {

1025 echo "Error: Poem parsing failed" >&2

1026 exit 1

1027 }

1028}

1032run_validate() {

1033 log_stage "$(symbol_success "✓") Stage 4/10: Validating poem data"

1035 if $DRY_RUN; then

1036 log_dry_run "luajit src/main.lua $DIR --validate-only $ASSETS_ARG"

1037 return 0

1038 fi

1040 luajit src/main.lua "$DIR" --validate-only $ASSETS_ARG || {

1041 echo "Error: Poem validation failed" >&2

1042 exit 1

1043 }

1044}

1049run_catalog_images() {

1050 log_stage "🖼️ Stage 5/10: Cataloging images"

1052 # Build verbose argument if enabled

1053 local VERBOSE_ARG=""

1054 $VERBOSE && VERBOSE_ARG="--verbose"

1056 if $DRY_RUN; then

1057 log_dry_run "luajit src/main.lua $DIR --catalog-only $VERBOSE_ARG $ASSETS_ARG $RANDOM_SEED_ARG"

1058 return 0

1059 fi

1061 luajit src/main.lua "$DIR" --catalog-only $VERBOSE_ARG $ASSETS_ARG $RANDOM_SEED_ARG || {

1062 echo "Error: Image cataloging failed" >&2

1063 exit 1

1064 }

1065}

1074emb_cache_dir() {

1075 local d

1076 d="$(luajit "$DIR/scripts/cache-dir" "$DIR" --model "$MODEL_NAME" "$@")"

1077 if [ -z "$d" ]; then

1078 echo "Error: could not resolve cache dir (scripts/cache-dir)" >&2

1079 exit 1

1080 fi

1081 echo "$d"

1082}

1086run_generate_embeddings() {

1087 log_stage "🤖 Stage 6/10: Generating embeddings via the inference server"

1089 # Convert model name for directory (embeddinggemma:latest -> embeddinggemma_latest)

1090 local model_dir_name="${MODEL_NAME//:/_}"

1091 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1092 local poems_file="$DIR/assets/poems.json"

1094 # Issue 10-016: Check both global and per-stage force flags

1095 local stage_force=$FORCE

1096 $FORCE_STAGE_6 && stage_force=true

1098 # Freshness check (Issue 10-050): skip ONLY when every poem already has an

1099 # embedding. The old test compared mtimes (embeddings.json newer than

1100 # poems.json) — which was wrong: a run that embedded 8160/8362 and then died

1101 # leaves a NEWER but INCOMPLETE embeddings.json, so mtime said "fresh, skip"

1102 # and the missing poems never got done. Counting entries is the honest

1103 # signal; incremental mode then fills only the gap, so it is cheap to re-run.

1104 if ! $stage_force && [ -f "$embeddings_file" ] && [ -f "$poems_file" ]; then

1105 # Count embeddings WITHOUT parsing the (large) JSON: each entry carries

1106 # exactly one "poem_index" key. (This counts error records too, so it can

1107 # only over-report completeness; incremental retries those anyway.)

1108 local emb_count

1109 emb_count=$(grep -o '"poem_index"' "$embeddings_file" | wc -l)

1110 local poem_count

1111 poem_count=$(luajit -e "

1112 package.path = '$DIR/?.lua;' .. package.path

1113 local dk = require('libs/dkjson')

1114 local f = io.open('$poems_file'); local d = dk.decode(f:read('*a')); f:close()

1115 print(#(d.poems or d))

1116 ")

1117 if [ -n "$poem_count" ] && [ "$poem_count" -gt 0 ] && [ "$emb_count" -ge "$poem_count" ]; then

1118 log_info " ⏭️ Embeddings complete ($emb_count/$poem_count), skipping..."

1119 return 0

1120 fi

1121 log_info " Embeddings incomplete ($emb_count/${poem_count:-?}) — running incremental to fill the gap..."

1122 fi

1124 local force_arg=""

1125 if $stage_force; then

1126 force_arg="--full-regen"

1127 else

1128 force_arg="--incremental"

1129 fi

1131 # Issue 10-017: Build Inference server argument

1132 local server_arg=""

1133 if [ -n "$INFERENCE_SERVER" ]; then

1134 server_arg="--server=$INFERENCE_SERVER"

1135 fi

1137 if $DRY_RUN; then

1138 log_dry_run "$DIR/generate-embeddings.sh $force_arg --model=$MODEL_NAME $server_arg $DIR"

1139 log_dry_run "luajit $DIR/src/generate-word-pages.lua $DIR --embeddings-only"

1140 return 0

1141 fi

1143 if [ -n "$INFERENCE_SERVER" ]; then

1144 log_info " Inference Server: $INFERENCE_SERVER"

1145 fi

1146 log_info " Model: $MODEL_NAME"

1147 log_info " Output: assets/embeddings/$model_dir_name/embeddings.json"

1148 log_info " Mode: $(if $FORCE; then echo 'full regeneration'; else echo 'incremental (skip existing)'; fi)"

1150 # Issue 10-028: Apply low priority to expensive embedding generation

1151 $NICE_PREFIX "$DIR/generate-embeddings.sh" $force_arg --model="$MODEL_NAME" $server_arg "$DIR" || {

1152 echo "Error: Embedding generation failed" >&2

1153 echo "Make sure the inference server is running with the $MODEL_NAME model" >&2

1154 exit 1

1155 }

1157 # Word embeddings used to run here, but the word-COLOR step inside

1158 # generate-word-pages needs color_embeddings.json, which is produced later by

1159 # run_generate_semantic_colors. Running words first made that step skip with

1160 # "no color embeddings found". Moved to run_generate_word_embeddings, called

1161 # AFTER colors in main.

1162}

1169run_generate_word_embeddings() {

1170 log_info " Generating word embeddings for word cloud..."

1171 # WORDCLOUD_WORDS carries either a number or the literal "all"; the generator

1172 # accepts both via --words (it treats "--words all" the same as "--all").

1173 local wordcloud_args=""

1174 if [ -n "$WORDCLOUD_WORDS" ]; then

1175 wordcloud_args="--words $WORDCLOUD_WORDS"

1176 fi

1177 $NICE_PREFIX luajit "$DIR/src/generate-word-pages.lua" "$DIR" --embeddings-only $wordcloud_args || {

1178 echo "Warning: Word embedding generation failed, continuing..." >&2

1179 }

1180}

1184run_generate_semantic_colors() {

1185 # Regenerate poem_colors.json if stale or missing

1186 # This runs BEFORE similarity matrix generation (Stage 6.5)

1187 # Requires: embeddings.json, color_embeddings.json

1188 # Respects: --force (skip freshness check), --dry-run (show actions only)

1190 local model_dir_name="${MODEL_NAME//:/_}"

1192 # Paths match what generate-embeddings.sh writes (see run_generate_embeddings above).

1193 # The stray assets/embeddings/embeddings/ directory on disk is a stale leftover from

1194 # before the model-name subfolder convention; it is not the real output location.

1195 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1196 local poem_colors_file="$(emb_cache_dir)/poem_colors.json"

1197 local color_embeddings_file="$(emb_cache_dir)/color_embeddings.json"

1199 # Embeddings must exist first (exit early if not - prevents confusing errors)

1200 if [ ! -f "$embeddings_file" ]; then

1201 log_verbose " Skipping semantic colors - embeddings not yet generated"

1202 return 0

1203 fi

1205 # color_embeddings.json is derived from the color palette (color_names +

1206 # color_associations in config.lua). It used to regenerate ONLY when the file was

1207 # missing, so editing the palette -- e.g. dropping gray as a cluster color -- had

1208 # no effect until someone deleted the cache by hand (and the config comment that

1209 # said "re-run stage 6.5 after editing" was quietly false). We now fingerprint the

1210 # palette and regenerate whenever it changes, so editing colors then re-running

1211 # actually takes effect. The fingerprint is a sorted, deterministic dump of the

1212 # palette -- no server needed to compute it.

1213 local palette_fp_file="$(emb_cache_dir)/color_palette.fingerprint"

1214 local current_palette_fp

1215 current_palette_fp=$(luajit -e "

1216 package.path = '$DIR/libs/?.lua;$DIR/src/?.lua;' .. package.path

1217 local config = require('config-loader').load()

1218 local names = {}

1219 for _, n in ipairs(config.color_names or {}) do names[#names+1] = n end

1220 table.sort(names)

1221 local parts = {}

1222 for _, n in ipairs(names) do

1223 local a = {}

1224 for _, w in ipairs((config.color_associations or {})[n] or {}) do a[#a+1] = w end

1225 table.sort(a)

1226 parts[#parts+1] = n .. '=' .. table.concat(a, ',')

1227 end

1228 io.write(table.concat(parts, '|'))

1229 ")

1230 local stored_palette_fp=""

1231 [ -f "$palette_fp_file" ] && stored_palette_fp=$(cat "$palette_fp_file")

1233 # Regenerate color embeddings if missing OR the palette changed since last time.

1234 if [ ! -f "$color_embeddings_file" ] || [ "$current_palette_fp" != "$stored_palette_fp" ]; then

1235 if [ -f "$color_embeddings_file" ]; then

1236 log_stage "🎨 Stage 6.5/10: Color palette changed -- regenerating color embeddings"

1237 else

1238 log_stage "🎨 Stage 6.5/10: Generating color embeddings (one-time)"

1239 fi

1241 if $DRY_RUN; then

1242 log_dry_run "luajit semantic-color-calculator (generate color embeddings)"

1243 # Still need to skip poem colors generation in dry run

1244 else

1245 log_info " $(symbol_warning "⚠️") Color embeddings not found, generating via the inference server..."

1246 # Issue 10-003 migrated color_names from config/semantic-colors.json (now deleted)

1247 # into config.lua, loaded via libs/config-loader.lua. Errors here are loud rather

1248 # than silent so a missing config doesn't propagate downstream as a confusing

1249 # "Failed to load required data files" in the next stage.

1250 luajit -e "

1251 package.path = '$DIR/libs/?.lua;$DIR/src/?.lua;' .. package.path

1252 local calc = require('semantic-color-calculator')

1253 local utils = require('utils')

1254 utils.init_assets_root({'$DIR'})

1256 -- Mirror the --server selection pattern used elsewhere in run.sh

1257 -- (see the interactive TUI block below). If INFERENCE_SERVER is empty

1258 -- the module falls back to config.lua's default_inference_server.

1259 -- The interactive flag is forwarded so that a typoed --server or

1260 -- --model triggers a 1/2 prompt only when the operator launched

1261 -- run.sh with -I; otherwise we hard-error.

1262 local inference = require('inference-server-config')

1263 inference.set_project_root('$DIR')

1264 inference.set_interactive_mode('$INTERACTIVE' == 'true')

1265 if '$INFERENCE_SERVER' ~= '' then

1266 inference.set_selected_server('$INFERENCE_SERVER')

1267 end

1269 local config = require('config-loader').load()

1270 if not config.color_names then

1271 error('config.lua is missing color_names (Issue 10-003 migration)')

1272 end

1273 -- Pass color_associations so each color's embedding is the mean

1274 -- of its essence words, not the bare color word (richer + the

1275 -- z-scored assignment is balanced). nil endpoint = use the

1276 -- selected server. Falls back to bare words if associations absent.

1277 local embeddings = calc.generate_color_embeddings(config.color_names, '$MODEL_NAME', nil, config.color_associations)

1278 if not next(embeddings) then

1279 error('Inference server returned no color embeddings')

1280 end

1281 local data = {embeddings = embeddings, generated_at = os.date('%Y-%m-%d %H:%M:%S'), model_name = '$MODEL_NAME'}

1282 utils.write_json_file('$color_embeddings_file', data)

1283 print('[INFO] Color embeddings saved: ' .. '$color_embeddings_file')

1284 " || {

1285 echo "Error: Color embedding generation failed" >&2

1286 exit 1

1287 }

1288 # Remember the palette we just built from, so the next run can tell

1289 # whether it changed (and skip this server round-trip when it hasn't).

1290 echo "$current_palette_fp" > "$palette_fp_file"

1291 fi

1292 fi

1294 # Issue 10-016: Check both global and per-stage force flags (Stage 6)

1295 local stage_force=$FORCE

1296 $FORCE_STAGE_6 && stage_force=true

1298 # Check freshness: poem_colors.json should be newer than embeddings.json

1299 # With --force or --force-stage 6: always regenerate regardless of freshness

1300 if ! $stage_force && [ -f "$poem_colors_file" ] && [ -f "$embeddings_file" ]; then

1301 # Poem colors depend on BOTH the poem embeddings AND the color centroids, so

1302 # they are only fresh when newer than both. Watching only embeddings.json

1303 # meant a palette change (which rewrites color_embeddings.json but not

1304 # embeddings.json) left poem_colors.json stale yet considered "fresh".

1305 if [ "$poem_colors_file" -nt "$embeddings_file" ] && [ "$poem_colors_file" -nt "$color_embeddings_file" ]; then

1306 log_info " ⏭️ Semantic colors are fresh (newer than embeddings + palette), skipping..."

1307 return 0

1308 fi

1309 log_verbose " poem_colors.json is stale (older than embeddings or palette), regenerating..."

1310 elif $stage_force; then

1311 log_verbose " --force specified, regenerating semantic colors..."

1312 fi

1314 log_stage "🎨 Stage 6b/10: Computing semantic colors (part of embeddings)"

1316 if $DRY_RUN; then

1317 log_dry_run "luajit semantic-color-calculator (poem colors regeneration)"

1318 return 0

1319 fi

1321 log_info " Input: $embeddings_file"

1322 log_info " Output: $poem_colors_file"

1324 # Regenerate poem colors using existing embeddings

1325 luajit -e "

1326 package.path = '$DIR/libs/?.lua;$DIR/src/?.lua;' .. package.path

1327 local calc = require('semantic-color-calculator')

1328 local utils = require('utils')

1329 utils.init_assets_root({'$DIR'})

1331 local poems_data = utils.read_json_file(utils.asset_path('poems.json'))

1332 local embeddings_data = utils.read_json_file('$embeddings_file')

1333 local color_embeddings_data = utils.read_json_file('$color_embeddings_file')

1335 if poems_data and embeddings_data and color_embeddings_data then

1336 calc.precompute_poem_colors(poems_data, embeddings_data, color_embeddings_data.embeddings, '$poem_colors_file')

1337 else

1338 error('Failed to load required data files')

1339 end

1340 " || {

1341 echo "Error: Semantic color generation failed" >&2

1342 exit 1

1343 }

1344}

1353run_augment_images() {

1354 log_stage "🖼️ Stage 6.7: Folding images into the embedding set (pseudo-embeddings)"

1355 local model_dir_name="${MODEL_NAME//:/_}"

1356 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1357 if [ ! -f "$embeddings_file" ]; then

1358 echo "Error: embeddings.json not found; run --generate-embeddings first" >&2

1359 exit 1

1360 fi

1361 if $DRY_RUN; then

1362 log_dry_run "luajit $DIR/src/augment-embeddings-with-images.lua $DIR"

1363 return 0

1364 fi

1365 $NICE_PREFIX luajit "$DIR/src/augment-embeddings-with-images.lua" "$DIR" || {

1366 echo "Error: image augmentation failed" >&2

1367 exit 1

1368 }

1369}

1373run_generate_similarity() {

1374 # GPU (Vulkan) is required: these are O(N^2) similarity calculations that make no

1375 # sense on a CPU, so the CPU route was removed (Issue 10-057). A missing GPU library

1376 # is a hard error with build instructions, never a slow fallback.

1377 if [ ! -f "$DIR/libs/vulkan-compute/build/libvkcompute.so" ]; then

1378 echo "Error: GPU library not found: libs/vulkan-compute/build/libvkcompute.so" >&2

1379 echo "Build it: cd libs/vulkan-compute && make" >&2

1380 exit 1

1381 fi

1382 log_stage "📊 Stage 7/10: Building similarity matrix with GPU (~5-10 min)"

1384 # Convert model name for directory

1385 local model_dir_name="${MODEL_NAME//:/_}"

1386 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1388 # Check if embeddings exist

1389 if [ ! -f "$embeddings_file" ]; then

1390 echo "Error: Embeddings file not found: $embeddings_file" >&2

1391 echo "Run --generate-embeddings first" >&2

1392 exit 1

1393 fi

1395 # Issue 10-016: Check both global and per-stage force flags (Stage 7)

1396 local stage_force=$FORCE

1397 $FORCE_STAGE_7 && stage_force=true

1399 # Issue 8-033: Check for individual similarity files instead of monolithic matrix

1400 local similarities_dir="$(emb_cache_dir)/similarities"

1401 local similarity_count=0

1402 if [ -d "$similarities_dir" ]; then

1403 similarity_count=$(find "$similarities_dir" -name "poem_*.json" 2>/dev/null | wc -l)

1404 fi

1406 # Freshness check: skip if we have all files and they're fresh

1407 if ! $stage_force && [ "$similarity_count" -ge 7797 ]; then

1408 # Check if any are older than embeddings (check newest file)

1409 local newest_similarity=$(find "$similarities_dir" -name "poem_*.json" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-)

1410 if [ -n "$newest_similarity" ] && [ "$newest_similarity" -nt "$embeddings_file" ]; then

1411 log_info " ⏭️ Similarity files are fresh ($similarity_count files newer than embeddings), skipping..."

1412 return 0

1413 fi

1414 fi

1416 local threads_arg=""

1417 if [ -n "$THREADS" ]; then

1418 threads_arg="--threads=$THREADS"

1419 fi

1421 if $DRY_RUN; then

1422 log_dry_run "luajit (GPU vk_similarity via libvkcompute.so) --generate-matrix $threads_arg"

1423 return 0

1424 fi

1426 log_info " Input: assets/embeddings/$model_dir_name/embeddings.json"

1427 log_info " Output: assets/embeddings/$model_dir_name/similarities/*.json (individual files)"

1429 # Issue 10-016: Convert stage_force to Lua boolean for Lua function calls

1430 local stage_force_lua="false"

1431 $stage_force && stage_force_lua="true"

1433 # GPU similarity generation using Vulkan compute shaders (the only route now)

1434 log_info " Mode: GPU-accelerated (Vulkan)"

1436 # Pass threads value to GPU similarity (defaults to 8 if not specified)

1437 local default_threads=8

1438 local threads_to_use=${THREADS:-$default_threads}

1439 log_info " CPU sorting threads: $threads_to_use"

1441 DIR="$DIR" luajit -e "

1442 package.path = '$DIR/?.lua;$DIR/?/init.lua;$DIR/libs/?.lua;' .. package.path

1443 local vk_sim = require('libs.vulkan-compute.lua.vk_similarity')

1444 -- Issue 10-057: size the rankings cache to exactly what THIS build shows

1445 -- per poem -- the ACTUAL pages it generates (the --pages value, else the

1446 -- config default minimum_pages -- NOT the storage ceiling

1447 -- max_pages_per_poem) times the poems shown per page. Everything is read

1448 -- at runtime from the run's flags + config; no hardcoded page counts. The

1449 -- HTML stage's loader regenerates if a later run ever needs more (the

1450 -- top_k stamp makes that detectable). The list is sorted nearest-first, so

1451 -- the top-K ARE precisely what the pages display.

1452 local _cfg = require('config-loader'); _cfg.set_project_root('$DIR')

1453 local _pag = _cfg.load().pagination

1454 if not _pag then error('config.pagination missing; cannot size the rankings cache') end

1455 local _pages = tonumber('$PAGES') or _pag.minimum_pages

1456 local _per_page = tonumber('$POEMS_PER_PAGE') or _pag.poems_per_page

1457 if not _pages or not _per_page then

1458 error('cannot resolve pages/poems_per_page to size the rankings cache')

1459 end

1460 local _top_k = _pages * _per_page

1461 -- Use TRUE parallel GPU computation (Issue 9-002 original design)

1462 local success = vk_sim.generate_similarity_matrix_gpu_parallel(

1463 '$(emb_cache_dir)/embeddings.json',

1464 '$MODEL_NAME',

1465 $stage_force_lua,

1466 $threads_to_use,

1467 _top_k

1468 )

1469 if not success then

1470 print('[GPU SIMILARITY ERROR] GPU generation failed')

1471 os.exit(1)

1472 end

1473 " || {

1474 echo "Error: GPU similarity generation failed" >&2

1475 exit 1

1476 }

1478 # Note: Pre-sorted similarity rankings cache is now generated automatically

1479 # by the GPU similarity engine (in-RAM, no file re-reading needed)

1480}

1484run_generate_diversity() {

1485 # GPU (Vulkan) is required: the diversity walk is O(N^2) GPU work, so the CPU route

1486 # was removed (Issue 10-057). A missing GPU library is a hard error, not a fallback.

1487 if [ ! -f "$DIR/libs/vulkan-compute/build/libvkcompute.so" ]; then

1488 echo "Error: GPU library not found: libs/vulkan-compute/build/libvkcompute.so" >&2

1489 echo "Build it: cd libs/vulkan-compute && make" >&2

1490 exit 1

1491 fi

1492 log_stage "🎲 Stage 8/10: Pre-computing diversity cache with GPU (~1 min)"

1494 # Convert model name for directory

1495 local model_dir_name="${MODEL_NAME//:/_}"

1496 local cache_file="$(emb_cache_dir --disk)/diversity_cache.json"

1497 local embeddings_file="$(emb_cache_dir)/embeddings.json"

1499 # Check if embeddings exist

1500 if [ ! -f "$embeddings_file" ]; then

1501 echo "Error: Embeddings file not found: $embeddings_file" >&2

1502 echo "Run --generate-embeddings first" >&2

1503 exit 1

1504 fi

1506 # Issue 10-016: Check both global and per-stage force flags (Stage 8)

1507 local stage_force=$FORCE

1508 $FORCE_STAGE_8 && stage_force=true

1510 # Freshness check: skip if cache newer than embeddings

1511 if ! $stage_force && [ -f "$cache_file" ]; then

1512 if [ "$cache_file" -nt "$embeddings_file" ]; then

1513 log_info " ⏭️ Diversity cache is fresh (newer than embeddings), skipping..."

1514 return 0

1515 fi

1516 fi

1518 log_info " Input: assets/embeddings/$model_dir_name/embeddings.json"

1519 log_info " Output: assets/embeddings/$model_dir_name/diversity_cache.json"

1521 # GPU diversity generation using Vulkan compute shaders (the only route now)

1522 log_info " Mode: GPU-accelerated (Vulkan)"

1524 if $DRY_RUN; then

1525 log_dry_run "$DIR/scripts/precompute-diversity-sequences-gpu $DIR"

1526 return 0

1527 fi

1529 # Issue 10-028: Apply low priority to expensive diversity generation.

1530 # The model is no longer passed via env here: the wrapper resolves it

1531 # through inference-server-config, which reads this run's overrides notepad

1532 # (tmp/run-overrides.lua, written above from --model) and falls back to

1533 # config.lua -- so the CLI override is honored without a per-stage env var.

1534 # Issue 10-057: pass the run's page settings so the wrapper caps each diversity

1535 # sequence to the SAME K the similarity cache and the HTML stage use.

1536 PAGES="$PAGES" POEMS_PER_PAGE="$POEMS_PER_PAGE" $NICE_PREFIX "$DIR/scripts/precompute-diversity-sequences-gpu" "$DIR" || {

1537 echo "Error: GPU diversity cache generation failed" >&2

1538 exit 1

1539 }

1540}

1544run_generate_html() {

1545 log_stage "🌐 Stage 9/10: Generating website HTML"

1547 # Issue 10-016: Check both global and per-stage force flags (Stage 9)

1548 local stage_force=$FORCE

1549 $FORCE_STAGE_9 && stage_force=true

1551 # Issue 10-024: Clear output directories when forcing regeneration

1552 # This prevents stale files with obsolete poem_index values from persisting

1553 # after poem re-extraction changes the poem_index assignments

1554 if $stage_force; then

1555 log_info " Clearing stale HTML files (--force)..."

1556 rm -f "$DIR/output/similar/"*.html 2>/dev/null

1557 rm -f "$DIR/output/different/"*.html 2>/dev/null

1558 rm -f "$DIR/output/chronological/"*.html 2>/dev/null

1559 fi

1561 local force_arg=""

1562 if $stage_force; then

1563 force_arg="--force"

1564 fi

1566 local threads_arg=""

1567 if [ -n "$THREADS" ]; then

1568 threads_arg="--threads $THREADS"

1569 fi

1571 # Issue 8-022: Pagination arguments

1572 local pages_arg=""

1573 if [ -n "$PAGES" ]; then

1574 pages_arg="--pages $PAGES"

1575 fi

1577 local poems_per_page_arg=""

1578 if [ -n "$POEMS_PER_PAGE" ]; then

1579 poems_per_page_arg="--poems-per-page $POEMS_PER_PAGE"

1580 fi

1582 local chrono_per_page_arg=""

1583 if [ -n "$CHRONO_PER_PAGE" ]; then

1584 chrono_per_page_arg="--chrono-per-page $CHRONO_PER_PAGE"

1585 fi

1587 if $DRY_RUN; then

1588 log_dry_run "$DIR/scripts/sync-page-templates $DIR (restore explore-page copy into input/pages/)"

1589 log_dry_run "luajit src/main.lua $DIR --html-only $force_arg $threads_arg $pages_arg $poems_per_page_arg $chrono_per_page_arg $ASSETS_ARG"

1590 log_dry_run "luajit $DIR/src/generate-gallery-pages.lua $DIR"

1591 log_dry_run "luajit $DIR/src/generate-source-browser.lua $DIR"

1592 return 0

1593 fi

1595 # Issue 11-005: restore the authored explore-page copy into the ephemeral

1596 # input/pages/ before generating. The canonical, version-controlled source is

1597 # page-templates/*.txt; input/ is wiped + re-synced from external sources each

1598 # run and does NOT carry this prose, so it is copied back in here. (Edit the

1599 # files in page-templates/ -- input/pages/ is overwritten from them.)

1600 "$DIR/scripts/sync-page-templates" "$DIR" || {

1601 echo "Error: failed to restore page templates into input/pages/" >&2

1602 exit 1

1603 }

1605 # Issue 10-028: Apply low priority to HTML generation (parallel processing)

1606 $NICE_PREFIX luajit src/main.lua "$DIR" --html-only $force_arg $threads_arg $pages_arg $poems_per_page_arg $chrono_per_page_arg $ASSETS_ARG || {

1607 echo "Error: HTML generation failed" >&2

1608 exit 1

1609 }

1611 # Issue 10-059: the word-cloud menu and per-word similarity pages moved to their

1612 # own stage 10 (run_generate_wordcloud). They run after this stage, so the

1613 # chronological pages main.lua just built are already present for their #poem links.

1615 # Issue 10-042: Build the image gallery (masonry pages per source + index +

1616 # chronological). It was previously a separate manual step, so the gallery

1617 # went stale -- it now regenerates with every HTML run from image-catalog.json.

1618 log_info " Generating image gallery..."

1619 $NICE_PREFIX luajit "$DIR/src/generate-gallery-pages.lua" "$DIR" || {

1620 echo "Warning: Gallery generation failed, continuing..." >&2

1621 }

1623 # Issue 10-052: Build the link-only source browser (code/issues/docs as HTML)

1624 # under output/source/. This is the "git push that builds a webpage" -- the

1625 # private monorepo never leaves the machine; whoever has the site link can

1626 # browse the source. It publishes an ALLOWLIST only (never the private input

1627 # corpus), so it is safe to ship with the rest of the site.

1628 log_info " Generating source browser..."

1629 $NICE_PREFIX luajit "$DIR/src/generate-source-browser.lua" "$DIR" || {

1630 echo "Warning: Source browser generation failed, continuing..." >&2

1631 }

1632 # NOTE: the downloadable zip is built at POST time by running

1633 # scripts/build-download-zip directly, not here -- it is a deploy artifact, and

1634 # there is no point regenerating a multi-GB archive on every local build. (The

1635 # site's links are document-relative, so there is no URL-conversion step before

1636 # upload; just upload output/ and build the zip.)

1637}

1646run_generate_wordcloud() {

1647 log_stage "🔤 Stage 10/10: Generating word-cloud menu and per-word pages"

1649 # Issue 10-059/10-061: wipe the per-word pages before regenerating. A word that

1650 # has fallen out of the cloud since the last build leaves an orphan page that the

1651 # generator never overwrites -- and an orphan from before a link-scheme change

1652 # ships BROKEN links (this is exactly how 134 stale "/similar-different/" pages

1653 # survived into a relative-path build). The pages are fully regenerated from the

1654 # current word set just below, so clearing every run (not only on --force) is

1655 # safe and is the only way to guarantee no stale orphans. Matches the principle

1656 # that each stage wipes its own output subdirectory before rebuilding it.

1657 if [ -d "$DIR/output/wordcloud" ]; then

1658 log_info " Clearing stale per-word pages before regeneration..."

1659 rm -f "$DIR/output/wordcloud/"*.html

1660 fi

1662 # Word-cloud arguments. WORDCLOUD_WORDS is a number or "all"; --words carries

1663 # either ("--words all" == every word, per the generators).

1664 local wordcloud_words_arg=""

1665 if [ -n "$WORDCLOUD_WORDS" ]; then

1666 wordcloud_words_arg="--words $WORDCLOUD_WORDS"

1667 fi

1669 # Issue 8-050d: Poems per word-cloud page

1670 local wordcloud_poems_arg=""

1671 if [ -n "$WORDCLOUD_POEMS" ]; then

1672 wordcloud_poems_arg="--poems-per-page $WORDCLOUD_POEMS"

1673 fi

1675 # Issue 10-036: thread chrono_per_page so the word-cloud poem links paginate to

1676 # the SAME chronological pages stage 9 built (separate processes must agree on

1677 # page size, or every #poem link lands on the wrong page).

1678 local chrono_per_page_arg=""

1679 if [ -n "$CHRONO_PER_PAGE" ]; then

1680 chrono_per_page_arg="--chrono-per-page $CHRONO_PER_PAGE"

1681 fi

1683 if $DRY_RUN; then

1684 log_dry_run "luajit $DIR/src/wordcloud-generator.lua $DIR $wordcloud_words_arg $chrono_per_page_arg $RANDOM_SEED_ARG"

1685 log_dry_run "luajit $DIR/src/generate-word-pages.lua $DIR --html-only $wordcloud_words_arg $wordcloud_poems_arg $chrono_per_page_arg"

1686 return 0

1687 fi

1689 # The word cloud IS the site's menu (and carries the live poem index), so a

1690 # failure here is fatal, not a warning -- there is no usable entry page without it.

1691 log_info " Generating word cloud menu..."

1692 $NICE_PREFIX luajit "$DIR/src/wordcloud-generator.lua" "$DIR" $wordcloud_words_arg $chrono_per_page_arg $RANDOM_SEED_ARG || {

1693 echo "Error: Word cloud menu generation failed" >&2

1694 exit 1

1695 }

1697 log_info " Generating word similarity pages..."

1698 $NICE_PREFIX luajit "$DIR/src/generate-word-pages.lua" "$DIR" --html-only $wordcloud_words_arg $wordcloud_poems_arg $chrono_per_page_arg || {

1699 echo "Error: Word similarity page generation failed" >&2

1700 exit 1

1701 }

1702}

2069if $GENERATE_EMBEDDINGS && ! $DRY_RUN; then

2070 log_info "Validating Inference server connectivity..."

2071 VALIDATION_RESULT=$(luajit -e "

2072 package.path = '$DIR/libs/?.lua;' .. package.path

2073 local inference = require('inference-server-config')

2074 if '$INFERENCE_SERVER' ~= '' then

2075 inference.set_selected_server('$INFERENCE_SERVER')

2076 end

2077 local server = inference.get_selected_server()

2078 local ok, msg = inference.validate_server(server)

2079 if ok then

2080 print('OK:' .. server.name .. ':' .. inference.build_host_url(server))

2081 else

2082 print('FAIL:' .. server.name .. ':' .. msg)

2083 end

2084 " 2>&1)

2086 if [[ "$VALIDATION_RESULT" == OK:* ]]; then

2087 SERVER_NAME=$(echo "$VALIDATION_RESULT" | cut -d: -f2)

2088 SERVER_URL=$(echo "$VALIDATION_RESULT" | cut -d: -f3-)

2089 log_info " ✓ Inference server '$SERVER_NAME' is reachable at $SERVER_URL"

2090 else

2091 # Server unreachable. Try to start it ourselves (and remember we

2092 # did, so the EXIT trap shuts it down again). If start succeeds,

2093 # re-validate to confirm /health is responsive before proceeding.

2094 SERVER_NAME=$(echo "$VALIDATION_RESULT" | cut -d: -f2)

2095 ERROR_MSG=$(echo "$VALIDATION_RESULT" | cut -d: -f3-)

2096 log_info " ✗ Inference server '$SERVER_NAME' not reachable: $ERROR_MSG"

2097 log_info " Attempting to start it via scripts/start-llamacpp-server.sh..."

2099 START_ARGS=("$DIR")

2100 if [ -n "$INFERENCE_SERVER" ]; then

2101 START_ARGS+=("--server=$INFERENCE_SERVER")

2102 fi

2103 if "$DIR/scripts/start-llamacpp-server.sh" "${START_ARGS[@]}"; then

2104 WE_STARTED_INFERENCE_SERVER=true

2106 # Re-validate to confirm the freshly-started server is responsive.

2107 VALIDATION_RESULT=$(luajit -e "

2108 package.path = '$DIR/libs/?.lua;' .. package.path

2109 local inference = require('inference-server-config')

2110 if '$INFERENCE_SERVER' ~= '' then

2111 inference.set_selected_server('$INFERENCE_SERVER')

2112 end

2113 local server = inference.get_selected_server()

2114 local ok, msg = inference.validate_server(server)

2115 if ok then

2116 print('OK:' .. server.name .. ':' .. inference.build_host_url(server))

2117 else

2118 print('FAIL:' .. server.name .. ':' .. msg)

2119 end

2120 " 2>&1)

2122 if [[ "$VALIDATION_RESULT" == OK:* ]]; then

2123 SERVER_NAME=$(echo "$VALIDATION_RESULT" | cut -d: -f2)

2124 SERVER_URL=$(echo "$VALIDATION_RESULT" | cut -d: -f3-)

2125 log_info " ✓ Inference server '$SERVER_NAME' started at $SERVER_URL"

2126 log_info " (will be shut down again when this run completes)"

2127 else

2128 ERROR_MSG=$(echo "$VALIDATION_RESULT" | cut -d: -f3-)

2129 echo -e "${RED}❌ ERROR: Started the inference server but it is still not reachable${NC}" >&2

2130 echo -e "${RED} $ERROR_MSG${NC}" >&2

2131 echo -e "${YELLOW}💡 Check ${NEOCITIES_LOG_DIR:-$DIR/tmp}/llamacpp-server.log for the server's own diagnostics${NC}" >&2

2132 exit 1

2133 fi

2134 else

2135 echo -e "${RED}❌ ERROR: Failed to start the inference server${NC}" >&2

2136 echo -e "${YELLOW}💡 Run ./scripts/start-llamacpp-server.sh manually for verbose output${NC}" >&2

2137 echo -e "${YELLOW}💡 Use --list-servers to see available servers${NC}" >&2

2138 echo -e "${YELLOW}💡 Use --server=NAME to select a different server${NC}" >&2

2139 exit 1

2140 fi

2141 fi