src/generate-word-pages.lua

25local function setup_dir_path(provided_dir)

26 if provided_dir then

27 return provided_dir

28 end

29 return "/mnt/mtwo/programming/ai-stuff/neocities-modernization"

106local parsed_dir, RUN_MODE, CLI_ALL_WORDS, CLI_MAX_WORDS, CLI_POEMS_PER_PAGE, CLI_CHRONO_PER_PAGE = parse_args(arg)

107local DIR = setup_dir_path(parsed_dir)

108package.path = DIR .. "/libs/?.lua;" .. DIR .. "/src/?.lua;" .. package.path

110local dkjson = require("dkjson")

111local utils = require("utils")

112local inference_config = require("inference-server-config")

115local fuzzy = require("fuzzy-computing")

118local config_loader = require("config-loader")

119config_loader.set_project_root(DIR)

120local unified_config = config_loader.load()

123local poem_bars = require("poem-bars")

128local progress = require("progress-display")

132local flat_html = require("flat-html-generator")

134utils.init_assets_root(arg)

141local wc = unified_config.word_cloud or {}

142local effective_max_words

143if CLI_ALL_WORDS then

144 effective_max_words = math.huge -- No limit

145elseif CLI_MAX_WORDS then

146 effective_max_words = CLI_MAX_WORDS

148 effective_max_words = wc.max_words or 200

152local effective_poems_per_page = CLI_POEMS_PER_PAGE or wc.poems_per_page or 50

167inference_config.set_project_root(DIR)

168local CONFIG = {

169 model_name = inference_config.get_selected_model(),

170 max_poems_per_page = 100, -- Poems per word page

171 max_pages_per_word = 1, -- For now, just one page per word

172 word_embeddings_file = "word_embeddings.json",

173 poems_per_word_page = effective_poems_per_page, -- Issue 8-050d: configurable via CLI/config

174 max_words = effective_max_words, -- Max words to process (from CLI or config)

175}

179local function cosine_similarity(vec1, vec2)

180 if not vec1 or not vec2 or #vec1 ~= #vec2 then

181 return 0

182 end

184 local dot_product = 0

185 local norm1 = 0

186 local norm2 = 0

188 for i = 1, #vec1 do

189 dot_product = dot_product + (vec1[i] * vec2[i])

190 norm1 = norm1 + (vec1[i] * vec1[i])

191 norm2 = norm2 + (vec2[i] * vec2[i])

192 end

194 norm1 = math.sqrt(norm1)

195 norm2 = math.sqrt(norm2)

197 if norm1 == 0 or norm2 == 0 then

198 return 0

199 end

201 return dot_product / (norm1 * norm2)

206local function load_word_embeddings_cache()

207 local cache_file = utils.embeddings_dir() .. "/" .. CONFIG.word_embeddings_file

208 local data = utils.read_json_file(cache_file)

209 return data and data.embeddings or {}

214local function save_word_embeddings_cache(embeddings)

215 local cache_file = utils.embeddings_dir() .. "/" .. CONFIG.word_embeddings_file

216 local data = {

217 embeddings = embeddings,

218 model = CONFIG.model_name,

219 generated = os.date("%Y-%m-%d %H:%M:%S"),

220 count = 0

221 }

222 for _ in pairs(embeddings) do data.count = data.count + 1 end

224 return utils.write_json_file(cache_file, data)

230local function load_color_embeddings()

231 local color_file = utils.embeddings_dir() .. "/color_embeddings.json"

232 local data = utils.read_json_file(color_file)

233 return data and data.embeddings or nil

245local function compute_color_ranking(word_embedding, color_embeddings)

246 if not word_embedding or not color_embeddings then

247 return {}

248 end

250 local ranking = {}

251 for color_name, color_embedding in pairs(color_embeddings) do

252 ranking[#ranking + 1] = {

253 color = color_name,

254 similarity = cosine_similarity(word_embedding, color_embedding),

255 }

256 end

257 table.sort(ranking, function(a, b) return a.similarity > b.similarity end)

258 return ranking

264local function load_word_colors_cache()

265 local cache_file = utils.embeddings_dir() .. "/word_colors.json"

266 local data = utils.read_json_file(cache_file)

267 if data and data.word_colors then

268 -- Convert array to lookup table for easy access

269 local lookup = {}

270 for _, entry in ipairs(data.word_colors) do

271 lookup[entry.word] = entry

272 end

273 return lookup

274 end

275 return {}

281local function save_word_colors_cache(word_colors_array)

282 local cache_file = utils.embeddings_dir() .. "/word_colors.json"

283 local data = {

284 word_colors = word_colors_array,

285 model = CONFIG.model_name,

286 generated = os.date("%Y-%m-%d %H:%M:%S"),

287 count = #word_colors_array

288 }

289 return utils.write_json_file(cache_file, data)

295local function compute_word_colors(word_embeddings)

296 local color_embeddings = load_color_embeddings()

297 if not color_embeddings then

298 -- Hard error, not a silent skip (author's call): the color embeddings are

299 -- produced by the semantic-color stage and MUST exist by the time word

300 -- colors are computed. Skipping just shipped colorless words while hiding

301 -- a real upstream problem (e.g. the cache written to a path this reader

302 -- does not look at -- see the CACHE_IN_RAM desync, Issue 10-054).

303 error("word color computation needs color embeddings, but "

304 .. utils.embeddings_dir() .. "/color_embeddings.json was not found. "

305 .. "The semantic-color stage must run before this AND must write where "

306 .. "this reads. (Poem coloring regenerates them when absent; word "

307 .. "coloring could be unified to do the same instead of erroring.)")

308 end

310 local word_colors = {}

311 local count = 0

312 for word, embedding in pairs(word_embeddings) do

313 local ranking = compute_color_ranking(embedding, color_embeddings)

314 -- ranking[1] is the winner (gray is a valid winner here -- the word pages and

315 -- other consumers keep using `color`). The full ranking rides along in

316 -- `colors` so the word cloud can choose a non-gray color for large words.

317 local best = ranking[1] or { color = "gray", similarity = 0 }

318 table.insert(word_colors, {

319 word = word,

320 color = best.color,

321 similarity = best.similarity,

322 colors = ranking

323 })

324 count = count + 1

325 end

327 -- Sort by word for consistent output

328 table.sort(word_colors, function(a, b) return a.word < b.word end)

330 utils.log_info(string.format("Computed semantic colors for %d words", count))

331 return word_colors

339local function balanced_color_select(candidates, color_embeddings, color_names, N)

340 -- Phase 2: Compute color affinities for each candidate

341 for _, candidate in ipairs(candidates) do

342 local best_color = "gray"

343 local best_color_sim = -1

344 candidate.color_sims = {}

345 for _, color_name in ipairs(color_names) do

346 local color_emb = color_embeddings[color_name]

347 if color_emb then

348 local sim = cosine_similarity(color_emb, candidate.embedding)

349 candidate.color_sims[color_name] = sim

350 if sim > best_color_sim then

351 best_color_sim = sim

352 best_color = color_name

353 end

354 end

355 end

356 candidate.best_color = best_color

357 candidate.best_color_sim = best_color_sim

358 end

360 -- Phase 3: Build color buckets (sorted by word_similarity descending)

361 local buckets = {}

362 for _, color_name in ipairs(color_names) do

363 buckets[color_name] = {}

364 end

365 for _, candidate in ipairs(candidates) do

366 table.insert(buckets[candidate.best_color], candidate)

367 end

368 for _, color_name in ipairs(color_names) do

369 table.sort(buckets[color_name], function(a, b)

370 return a.word_similarity > b.word_similarity

371 end)

372 end

374 -- Phase 4: Balanced round-robin selection

375 -- Give priority to colors with lowest cumulative color-similarity totals

376 local cumulative = {}

377 local bucket_idx = {} -- next pick index per color

378 for _, color_name in ipairs(color_names) do

379 cumulative[color_name] = 0

380 bucket_idx[color_name] = 1

381 end

383 local selected = {}

384 while #selected < N do

385 -- Find color with lowest cumulative score that still has candidates

386 local pick_color = nil

387 local lowest_cum = math.huge

388 local most_remaining = -1

389 for _, color_name in ipairs(color_names) do

390 local remaining = #buckets[color_name] - bucket_idx[color_name] + 1

391 if remaining > 0 then

392 local cum = cumulative[color_name]

393 -- Tiebreak: prefer color with more remaining candidates

394 if cum < lowest_cum or (cum == lowest_cum and remaining > most_remaining) then

395 lowest_cum = cum

396 pick_color = color_name

397 most_remaining = remaining

398 end

399 end

400 end

402 if not pick_color then break end -- all buckets exhausted

404 -- Pop top candidate from this color's bucket

405 local idx = bucket_idx[pick_color]

406 local poem = buckets[pick_color][idx]

407 bucket_idx[pick_color] = idx + 1

409 -- Track cumulative color similarity (high-affinity colors "spend" budget faster)

410 cumulative[pick_color] = cumulative[pick_color] + poem.best_color_sim

412 table.insert(selected, poem)

413 end

415 return selected

422local function compute_centroid(poems, poem_lookup)

423 if not poems or #poems == 0 then return nil end

425 -- Find dimension from first valid embedding

426 local dim = nil

427 for _, entry in ipairs(poems) do

428 local poem_id = tostring(entry.poem and entry.poem.poem_index)

429 local emb = poem_lookup[poem_id]

430 if emb then

431 dim = #emb

432 break

433 end

434 end

435 if not dim then return nil end

437 -- Initialize centroid to zeros

438 local centroid = {}

439 for d = 1, dim do centroid[d] = 0 end

441 -- Sum embeddings of selected poems

442 local count = 0

443 for _, entry in ipairs(poems) do

444 local poem_id = tostring(entry.poem and entry.poem.poem_index)

445 local emb = poem_lookup[poem_id]

446 if emb then

447 for d = 1, dim do centroid[d] = centroid[d] + emb[d] end

448 count = count + 1

449 end

450 end

452 if count == 0 then return nil end

454 -- Average

455 for d = 1, dim do centroid[d] = centroid[d] / count end

457 return centroid

464local function find_closest_poem_to_centroid(centroid, poem_lookup, poems_by_index)

465 if not centroid then return nil end

467 local best_poem = nil

468 local best_similarity = -1

470 for poem_id_str, poem_embedding in pairs(poem_lookup) do

471 local sim = cosine_similarity(centroid, poem_embedding)

472 if sim > best_similarity then

473 best_similarity = sim

474 best_poem = poems_by_index[tonumber(poem_id_str)]

475 end

476 end

478 return best_poem

484local function get_word_list(poems_data, stop_words, min_occurrences, max_words, min_word_length)

485 local word_counts = {}

487 for _, poem in ipairs(poems_data.poems or {}) do

488 local content = poem.content or ""

489 for word in content:gmatch("[%w]+") do

490 local normalized = word:lower()

491 if #normalized >= min_word_length

492 and not stop_words[normalized]

493 and not normalized:match("^%d+$") then

494 word_counts[normalized] = (word_counts[normalized] or 0) + 1

495 end

496 end

497 end

499 -- Filter and sort

500 local filtered = {}

501 for word, count in pairs(word_counts) do

502 if count >= min_occurrences then

503 table.insert(filtered, {word = word, count = count})

504 end

505 end

506 -- Sort by count descending, with an ALPHABETICAL tiebreaker. The tiebreak

507 -- is load-bearing, not cosmetic: `filtered` is built by iterating a Lua

508 -- hash (pairs), whose order differs from process to process, and

509 -- table.sort is not stable. Without a deterministic tiebreak, two separate

510 -- runs (stage 6 generating embeddings, the HTML stage consuming them) sort

511 -- ties differently, so the max_words cutoff keeps DIFFERENT words each run

512 -- -- which is exactly how a word ends up in the HTML list with no

513 -- embedding ("Missing embedding for word X"). Sorting ties by word makes

514 -- the cutoff identical across processes, closing that gap.

515 table.sort(filtered, function(a, b)

516 if a.count ~= b.count then return a.count > b.count end

517 return a.word < b.word

518 end)

520 -- Limit to max_words

521 local result = {}

522 for i = 1, math.min(#filtered, max_words) do

523 result[i] = filtered[i].word

524 end

526 return result

532local function load_stop_words()

533 local stop_words = {}

534 local wc = unified_config.word_cloud or {}

535 for _, word in ipairs(wc.stop_words or {}) do

536 stop_words[word:lower()] = true

537 end

538 return stop_words

543local function build_poem_embeddings_lookup(embeddings_data)

544 local lookup = {}

545 if not embeddings_data or not embeddings_data.embeddings then

546 return lookup

547 end

549 for _, entry in ipairs(embeddings_data.embeddings) do

550 if entry.id and entry.embedding then

551 lookup[tostring(entry.id)] = entry.embedding

552 end

553 end

555 return lookup

564local function format_poem_for_word_page(poem, rank, similarity, poem_colors, color_config, chrono_map, chrono_page_map)

565 local poem_idx = poem.poem_index or 0

567 -- Get semantic color for this poem (default to gray)

568 local poem_color_data = poem_colors and poem_colors[poem_idx]

569 local semantic_color = poem_color_data and poem_color_data.color or "gray"

570 local hex_color = color_config and color_config[semantic_color] or "#888888"

572 -- Check if golden poem (metadata-based detection)

573 local is_golden = poem.metadata and poem.metadata.is_golden_poem

575 -- Use CHRONOLOGICAL position for progress bar (not similarity score)

576 -- This matches similar/different pages and helps orient the reader in the story

577 -- Issue 8-045: Use timeline_progress (time-based) instead of position-based

578 local chrono_info = chrono_map and chrono_map[poem_idx] or {position = 1, total_poems = 1, timeline_progress = 50}

579 local progress_pct = chrono_info.timeline_progress or ((chrono_info.position / chrono_info.total_poems) * 100)

581 -- Calculate progress bar chars

582 -- Regular: 83 chars total, Golden: 82 interior + 2 corners = 84 total

583 local total_bar_chars = is_golden and 82 or 83

584 local progress_chars = math.floor((progress_pct / 100) * total_bar_chars)

585 local remaining_chars = total_bar_chars - progress_chars

587 -- Top progress bar from the shared poem-bars module (canonical geometry,

588 -- same as the similar/different + chronological pages).

589 poem_bars.configure(color_config)

590 local colored_progress = poem_bars.progress_dashes(

591 { percentage = progress_pct }, semantic_color, is_golden, "top", false).visual

593 -- Navigation links

594 local base_path = ".."

595 local similar_link = string.format("<a href='%s/similar/%04d-01.html'>similar</a>", base_path, poem_idx)

596 local different_link = string.format("<a href='%s/different/%04d-01.html'>different</a>", base_path, poem_idx)

597 -- Anchor must match the spans the chronological pages emit, which are

598 -- get_poem_anchor_id() = "poem-<poem_index>". The old "poem-CATEGORY-ID"

599 -- form never matched any anchor, so the chronological link landed at the

600 -- top of the page instead of the poem.

601 local anchor_id = string.format("poem-%d", poem_idx)

602 -- Issue 10-036: Use chrono_page_map for correct paginated link (index.html is redirect that loses anchors)

603 local chrono_page = chrono_page_map and chrono_page_map[poem_idx] or "01"

604 local chrono_link = string.format("<a href='%s/chronological/%s.html#%s'>chronological</a>", base_path, chrono_page, anchor_id)

606 -- Word-wrap content to 80 chars

607 local content = poem.content or ""

608 content = content:gsub("&", "&amp;"):gsub("<", "&lt;"):gsub(">", "&gt;")

610 local wrapped_lines = {}

612 -- Handle content warning from poem.content_warning (ActivityPub CW)

613 if poem.content_warning and poem.content_warning ~= "" then

614 local cw_display = "CW: " .. poem.content_warning

615 local box_width = math.min(math.max(#cw_display, 20), 76)

616 local padded_cw = cw_display .. string.rep(" ", box_width - #cw_display)

617 table.insert(wrapped_lines, " ┌" .. string.rep("─", box_width + 2) .. "┐")

618 table.insert(wrapped_lines, " │ " .. padded_cw .. " │")

619 table.insert(wrapped_lines, " └" .. string.rep("─", box_width + 2) .. "┘")

620 table.insert(wrapped_lines, "")

621 table.insert(wrapped_lines, "")

622 end

624 -- Handle in-content CW: patterns

625 local main_content = content

626 local cw_match = content:match("^%s*[Cc][Ww]%s*:(.-)[\n\r]")

627 if not cw_match then

628 cw_match = content:match("^%s*[Cc]ontent [Ww]arning%s*:(.-)[\n\r]")

629 end

630 if cw_match then

631 local cw_text = cw_match:match("^%s*(.-)%s*$")

632 main_content = content:gsub("^%s*[Cc][Ww]%s*:[^\n\r]*[\n\r]?", "")

633 main_content = main_content:gsub("^%s*[Cc]ontent [Ww]arning%s*:[^\n\r]*[\n\r]?", "")

634 if cw_text and #cw_text > 0 then

635 local cw_display = "CW: " .. cw_text

636 local box_width = math.min(math.max(#cw_display, 20), 76)

637 local padded_cw = cw_display .. string.rep(" ", box_width - #cw_display)

638 table.insert(wrapped_lines, " ┌" .. string.rep("─", box_width + 2) .. "┐")

639 table.insert(wrapped_lines, " │ " .. padded_cw .. " │")

640 table.insert(wrapped_lines, " └" .. string.rep("─", box_width + 2) .. "┘")

641 table.insert(wrapped_lines, "")

642 end

643 end

645 -- Word-wrap paragraphs

646 for para in (main_content .. "\n"):gmatch("(.-)\n") do

647 if para == "" then

648 table.insert(wrapped_lines, "")

649 else

650 local current_line = ""

651 for word in para:gmatch("%S+") do

652 if #current_line + #word + 1 <= 80 then

653 current_line = current_line .. (current_line ~= "" and " " or "") .. word

654 else

655 if current_line ~= "" then table.insert(wrapped_lines, " " .. current_line) end

656 current_line = word

657 end

658 end

659 if current_line ~= "" then table.insert(wrapped_lines, " " .. current_line) end

660 end

661 end

663 -- Apply golden side borders if needed

664 if is_golden then

665 local golden_lines = {}

666 local colored_wall = string.format('<font color="%s"><b>║</b></font>', hex_color)

667 local CONTENT_WIDTH = 80

669 local function utf8_char_count(str)

670 return #(str:gsub("[\128-\191]", ""))

671 end

673 for _, line in ipairs(wrapped_lines) do

674 local line_content = line:match("^%s*(.*)$") or line

675 local visible_content = line_content:gsub("<[^>]+>", "")

676 local visible_length = utf8_char_count(visible_content)

677 local padded_content

678 if visible_length >= CONTENT_WIDTH then

679 padded_content = line_content

680 else

681 padded_content = line_content .. string.rep(" ", CONTENT_WIDTH - visible_length)

682 end

683 table.insert(golden_lines, colored_wall .. " " .. padded_content .. " │")

684 end

685 wrapped_lines = golden_lines

686 end

688 -- Helper to colorize box characters based on progress

689 local function color_char(char, pos)

690 if progress_chars > pos then

691 return string.format('<font color="%s"><b>%s</b></font>', hex_color, char)

692 end

693 return char

694 end

696 -- Navigation box + bottom bar from the shared poem-bars module. The old

697 -- inline copy had drifted (golden junctions at 9/70 instead of 10/71), which

698 -- is exactly why word-cloud golden poems were mangled.

699 local nav_top, nav_mid

700 if is_golden then

701 nav_top = poem_bars.golden_corner_box_separator(hex_color, progress_chars)

702 nav_mid = poem_bars.golden_corner_box_nav_line(similar_link, different_link, chrono_link, hex_color, progress_chars)

703 else

704 nav_top = poem_bars.corner_box_top(progress_chars, hex_color)

705 nav_mid = poem_bars.corner_box_nav_line(similar_link, different_link, chrono_link, progress_chars, hex_color)

706 end

708 local bottom_line = poem_bars.progress_dashes(

709 { percentage = progress_pct }, semantic_color, is_golden, "bottom", true).visual

711 -- Generate poem identifier (same format as similar/different pages)

712 -- Format: " -> file: fediverse/1234" or " -> file: notes/myfile"

713 local category = poem.category or "unknown"

714 local filename

715 if category == "notes" and poem.metadata and poem.metadata.source_file then

716 filename = poem.metadata.source_file

717 else

718 filename = tostring(poem.id or "unknown")

719 end

720 local poem_identifier = " -> file: " .. category .. "/" .. filename

722 -- Build final output

723 local output = {}

724 table.insert(output, colored_progress)

725 if is_golden then

726 -- The header (" -> file:") and the blank line below it belong INSIDE the

727 -- golden box, with the same ║ ... │ walls as every other line, so the box

728 -- has consistent borders and uniform line width (no special-spaced gap).

729 local function golden_line(content)

730 local visible = content:gsub("<[^>]+>", "")

731 local vlen = #(visible:gsub("[\128-\191]", ""))

732 local padded = content .. string.rep(" ", math.max(0, 80 - vlen))

733 return string.format('<font color="%s"><b>║</b></font> %s │', hex_color, padded)

734 end

735 table.insert(output, golden_line((poem_identifier:gsub("^%s+", ""))))

736 table.insert(output, golden_line(""))

737 else

738 table.insert(output, poem_identifier)

739 table.insert(output, "")

740 end

741 table.insert(output, table.concat(wrapped_lines, "\n"))

742 table.insert(output, nav_top)

743 table.insert(output, nav_mid)

744 table.insert(output, bottom_line)

746 return table.concat(output, "\n")

757local function generate_word_page(word, ranked_poems, output_dir, poems_per_page, poem_colors, color_config, chrono_map, word_hex_color, chrono_center_link, chrono_page_map)

758 local safe_word = word:lower():gsub("[^%w]", "")

759 local output_file = output_dir .. "/wordcloud/" .. safe_word .. ".html"

761 -- Ensure directory exists

762 os.execute('mkdir -p "' .. output_dir .. '/wordcloud"')

764 -- Take top N poems

765 local top_poems = {}

766 for i = 1, math.min(poems_per_page, #ranked_poems) do

767 top_poems[i] = ranked_poems[i]

768 end

770 -- Issue 8-050c: Use word's semantic color for header (default to gray if not provided)

771 local header_color = word_hex_color or "#888888"

773 -- Issue 8-050e: Use centroid-based chronological link if provided, else default

774 local base_path = ".."

775 local chrono_link = chrono_center_link or (base_path .. "/chronological/index.html")

777 -- Generate HTML

778 -- Issue 16-010: Added font style for Hack Nerd Font font-stack

779 -- Same centering CSS the similar/different/chronological pages use: each

780 -- <pre> centers as an inline-block (text stays left), so the poem column

781 -- lands on the page centerline even when an attached image is wider.

782 local font_style = [[<style>body, pre { font-family: 'Hack Nerd Font', 'Hack', 'Fira Code', 'JetBrains Mono', 'Cascadia Code', 'Consolas', 'Monaco', 'Liberation Mono', 'Courier New', monospace; }

783td { text-align: center; } pre { display: inline-block; text-align: left; margin: 0 auto; } img, video, audio { margin-left: auto; margin-right: auto; }</style>]]

784 local html_parts = {}

785 table.insert(html_parts, string.format([[<!DOCTYPE html>

786<html>

787<head>

788<meta charset="UTF-8">

789<title>Poems similar to: %s</title>

790%s</head>

791<body bgcolor="#000000" text="#FFFFFF" link="#6699FF" vlink="#9966FF">

792<center>

793<h1>Poems similar to: <i><font color="%s">%s</font></i></h1>

794<p>Top %d poems ranked by semantic similarity (progress bar shows chronological position)</p>

795<!-- Issue 16-010: Changed main.html to wordcloud.html (main.html doesn't exist) -->

796<p><a href="%s/wordcloud.html">Menu</a> │ <a href="%s">Chronological</a></p>

797</center>

798<hr>

799<table align="center"><tr><td>

800<pre>

801]], word, font_style, header_color, word, #top_poems, base_path, chrono_link))

803 -- Add ranked poems using box-drawing format

804 -- Issue 10-036: Pass chrono_page_map for correct per-poem pagination links

805 for i, entry in ipairs(top_poems) do

806 local formatted = format_poem_for_word_page(entry.poem, i, entry.similarity, poem_colors, color_config, chrono_map, chrono_page_map)

807 table.insert(html_parts, formatted)

808 table.insert(html_parts, "\n")

809 end

811 table.insert(html_parts, [[</pre>

812</td></tr></table>

813</body>

814</html>

815]])

817 local html = table.concat(html_parts)

818 return utils.write_file(output_file, html)

825function M.generate_word_embeddings(options)

826 options = options or {}

828 -- Check inference server availability

829 -- Issue 10-017: Use build_host_url() instead of deprecated OLLAMA_ENDPOINT

830 local endpoint = inference_config.build_host_url()

831 utils.log_info("Using inference endpoint: " .. endpoint)

833 -- Load poems for word extraction

834 local poems_file = utils.asset_path("poems.json")

835 local poems_data = utils.read_json_file(poems_file)

836 if not poems_data then

837 utils.log_error("Could not load poems.json")

838 return nil

839 end

841 -- Get word list (using CONFIG.max_words from CLI or config)

842 local stop_words = load_stop_words()

843 local words = get_word_list(poems_data, stop_words, 5, CONFIG.max_words, 3)

844 utils.log_info(string.format("Processing %d words", #words))

846 -- Load cached word embeddings

847 local word_embeddings = load_word_embeddings_cache()

848 local cache_hits = 0

849 local cache_misses = 0

851 -- Issue 10-050: collect the words missing from cache, then embed them all in

852 -- one batched + (sub-)batched call instead of one curl per word. Words are

853 -- single tokens so chunking is a no-op; embed_texts_with_chunking still

854 -- splits the request into BATCH_SIZE-sized round trips. endpoint + prompt

855 -- formatter are passed so the prefix matches the poem embeddings (required

856 -- for the word-to-poem cosine comparison to be meaningful).

857 local missing = {}

858 for _, word in ipairs(words) do

859 if not word_embeddings[word] then

860 missing[#missing + 1] = word

861 end

862 end

863 cache_hits = #words - #missing

865 if #missing > 0 then

866 utils.log_info(string.format("Embedding %d missing words (batched)...", #missing))

867 -- Issue 10-065: this is minutes of network round trips with nothing on

868 -- screen -- the longest silent stretch in the pipeline. The batch

869 -- embedder now takes a progress callback, invoked once per request, so

870 -- the wait has a bar like every other stage. Throttling is unnecessary

871 -- here: a callback fires per REQUEST (dozens to hundreds of words each),

872 -- not per word, so the redraw rate is already low.

873 local vectors = fuzzy.embed_texts_with_chunking(missing, CONFIG.model_name, {

874 endpoint = endpoint,

875 format_fn = inference_config.format_embedding_prompt,

876 on_progress = function(done, total)

877 progress.update(" 🔡 Word embeddings", done, total)

878 end

879 })

880 progress.finish()

881 if vectors then

882 for k, word in ipairs(missing) do

883 local embedding = vectors[k]

884 if embedding and type(embedding) == "table" and #embedding > 0 then

885 word_embeddings[word] = embedding

886 cache_misses = cache_misses + 1

887 -- Periodic checkpoint so a crash mid-run keeps prior work.

888 if cache_misses % 50 == 0 then

889 save_word_embeddings_cache(word_embeddings)

890 end

891 else

892 utils.log_warn(string.format("Failed to embed word '%s'", word))

893 end

894 end

895 else

896 utils.log_warn("Batch word embedding failed (inference server unreachable?)")

897 end

898 end

900 -- Save final cache

901 save_word_embeddings_cache(word_embeddings)

902 utils.log_info(string.format("Word embeddings: %d cached, %d newly generated", cache_hits, cache_misses))

904 -- Issue 8-050a: Compute and save semantic colors for all words

905 local word_colors = compute_word_colors(word_embeddings)

906 if word_colors then

907 save_word_colors_cache(word_colors)

908 utils.log_info(string.format("Saved semantic colors for %d words to word_colors.json", #word_colors))

909 end

911 return cache_hits + cache_misses

919function M.generate_word_html(options)

920 options = options or {}

921 local output_dir = options.output_dir or (DIR .. "/output")

923 -- Load poems

924 local poems_file = utils.asset_path("poems.json")

925 local poems_data = utils.read_json_file(poems_file)

926 if not poems_data then

927 utils.log_error("Could not load poems.json")

928 return nil

929 end

931 -- Load poem embeddings

932 local embeddings_file = utils.embeddings_dir() .. "/embeddings.json"

933 local embeddings_data = utils.read_json_file(embeddings_file)

934 if not embeddings_data then

935 utils.log_error("Could not load poem embeddings - run --generate-embeddings first")

936 return nil

937 end

938 local poem_lookup = build_poem_embeddings_lookup(embeddings_data)

940 -- Load word embeddings (must exist from Stage 6)

941 local word_embeddings = load_word_embeddings_cache()

942 local word_count = 0

943 for _ in pairs(word_embeddings) do word_count = word_count + 1 end

945 if word_count == 0 then

946 utils.log_error("No word embeddings found - run --embeddings-only first")

947 return nil

948 end

949 utils.log_info(string.format("Loaded %d word embeddings", word_count))

951 -- Issue 8-043c: Load poem colors for semantic coloring

952 -- Issue 10-034: Fixed path - poem_colors.json is in embeddings directory, not assets root

953 local poem_colors_file = utils.embeddings_dir() .. "/poem_colors.json"

954 local poem_colors_data = utils.read_json_file(poem_colors_file)

955 -- poem_colors.json stores a plain ARRAY whose position IS the poem_index

956 -- (entries carry color/similarity but NO poem_index field). flat-html

957 -- reads it positionally (poem_colors[poem_index]); we must do the same.

958 -- The old code keyed on entry.poem_index -- always nil -- so the table came

959 -- out empty and every word-page progress bar fell back to gray. Reading the

960 -- array directly is what makes those bars match the similar/different pages.

961 local poem_colors = (poem_colors_data and poem_colors_data.poem_colors) or {}

962 if not (poem_colors_data and poem_colors_data.poem_colors) then

963 utils.log_warn("No poem colors found - using default gray")

964 end

966 -- Issue 8-050a: Load word colors for per-word semantic coloring

967 local word_colors = load_word_colors_cache()

968 local word_color_count = 0

969 for _ in pairs(word_colors) do word_color_count = word_color_count + 1 end

970 if word_color_count == 0 then

971 utils.log_warn("No word colors found - run --embeddings-only to generate them")

972 end

974 -- Issue 8-050b: Load color embeddings for balanced color selection

975 local color_embeddings = load_color_embeddings()

976 local use_balanced_selection = color_embeddings ~= nil

977 if not use_balanced_selection then

978 utils.log_warn("No color embeddings found - using pure similarity ranking")

979 end

981 -- Issue 8-050b: Get ordered color names from config

982 local color_names = unified_config.color_names

983 or {"red", "blue", "green", "purple", "orange", "yellow", "gray"}

985 -- Issue 8-043c: Load color configuration from unified config

986 local color_config = unified_config.colors or {

987 red = "#FF6B6B",

988 orange = "#FFA94D",

989 yellow = "#FFE066",

990 green = "#69DB7C",

991 cyan = "#38D9A9",

992 blue = "#74C0FC",

993 indigo = "#748FFC",

994 violet = "#DA77F2",

995 gray = "#868E96"

996 }

998 -- Issue 8-043c: Compute chronological mapping for progress bars

999 -- This maps poem_index → {position, total_poems} for timeline orientation

1000 -- Issue 8-050e: Also builds chrono_page_map for centroid-based navigation

1001 local chrono_map = {}

1002 local chrono_page_map = {} -- poem_index → page string ("01", "02", etc.)

1003 do

1004 -- Reuse the chronological-page generator's OWN mapping instead of a second

1005 -- inline copy. The old copy sorted by the raw creation_date string with no

1006 -- tiebreaker and a 500/page default, so it disagreed with the actual

1007 -- chronological pagination (timestamp sort + original-index tiebreaker +

1008 -- config page size) -> links jumped to the wrong page and never scrolled.

1009 -- The page size comes from resolve_chrono_per_page() (the build's

1010 -- --chrono-per-page, else config). It MUST match what the chronological

1011 -- pages were built with; a wrong size is exactly what broke these links,

1012 -- so an absent value hard-errors rather than guessing (Issue 10-036).

1013 local per_page = resolve_chrono_per_page()

1014 local mapping = flat_html.compute_chronological_mapping(poems_data, per_page)

1015 local total_poems = 0

1016 for poem_index, info in pairs(mapping) do

1017 chrono_map[poem_index] = {

1018 position = info.position,

1019 total_poems = info.total_poems,

1020 -- Carry the time-based progress so the word-page bars match the

1021 -- similar/different/chronological pages exactly (not position-based).

1022 timeline_progress = info.timeline_progress,

1023 }

1024 chrono_page_map[poem_index] = string.format("%02d", info.page_number)

1025 total_poems = info.total_poems

1026 end

1027 utils.log_info(string.format("Built chronological mapping for %d poems (%d per page, shared)", total_poems, per_page))

1028 end

1030 -- Build poem index lookup

1031 local poems_by_index = {}

1032 for _, poem in ipairs(poems_data.poems) do

1033 if poem.poem_index then

1034 poems_by_index[poem.poem_index] = poem

1035 end

1036 end

1038 -- Get word list (same as embedding generation to ensure consistency)

1039 local stop_words = load_stop_words()

1040 local words = get_word_list(poems_data, stop_words, 5, CONFIG.max_words, 3)

1042 -- Generate pages for each word

1043 local pages_generated = 0

1044 for i, word in ipairs(words) do

1045 -- Issue 10-065: update OUTSIDE the has-an-embedding test below, so the

1046 -- bar tracks position in the word list rather than only the words that

1047 -- happened to produce a page -- otherwise it stalls silently through any

1048 -- run of words with no embedding.

1049 --

1050 -- Replaces a hand-rolled io.write("\r...") that bypassed this library.

1051 -- Three things were wrong with that. It emitted NO newlines, so under

1052 -- --debug -- where stdout is a pipe to scripts/fsync-logger, which reads

1053 -- line by line -- a whole run's progress accumulated into one enormous

1054 -- unterminated line instead of the durable per-line history --debug

1055 -- exists to produce. It drew unconditionally when piped, where every

1056 -- other stage stays quiet. And it looked nothing like the other bars.

1057 local step = (progress.mode() == 2) and 100 or 25

1058 if i % step == 0 or i == #words then

1059 progress.update(" 🔤 Word pages", i, #words, word)

1060 end

1062 local word_embedding = word_embeddings[word]

1063 if word_embedding then

1065 -- Issue 8-050b: Build candidate pool with embeddings preserved

1066 -- Phase 1: Rank ALL poems by word similarity

1067 local candidates = {}

1068 for poem_id_str, poem_embedding in pairs(poem_lookup) do

1069 local poem_id = tonumber(poem_id_str)

1070 local poem = poems_by_index[poem_id]

1071 if poem and poem_embedding then

1072 local word_sim = cosine_similarity(word_embedding, poem_embedding)

1073 table.insert(candidates, {

1074 poem = poem,

1075 embedding = poem_embedding,

1076 word_similarity = word_sim,

1077 similarity = word_sim -- for generate_word_page compatibility

1078 })

1079 end

1080 end

1082 -- Sort by word similarity (descending)

1083 table.sort(candidates, function(a, b)

1084 return a.word_similarity > b.word_similarity

1085 end)

1087 -- Issue 8-050b (revised): relevance first, THEN color spread.

1088 -- The page always shows the top-N MOST RELEVANT poems by similarity;

1089 -- balanced_color_select is handed exactly those N (not a 7N pool),

1090 -- so it keeps the whole relevant set and only REORDERS it to spread

1091 -- the colors across the page. The earlier 7N-pool version let color

1092 -- balancing DISPLACE strong matches with weaker color-diverse ones,

1093 -- which is why a "god" search surfaced unrelated poems.

1094 local ranked_poems

1095 if use_balanced_selection then

1096 local pool_size = math.min(#candidates, CONFIG.poems_per_word_page)

1097 local pool = {}

1098 for j = 1, pool_size do pool[j] = candidates[j] end

1100 -- Reorder the top-N relevant poems for color spread (keeps all N).

1101 ranked_poems = balanced_color_select(

1102 pool, color_embeddings, color_names, CONFIG.poems_per_word_page)

1103 else

1104 -- Fallback: pure similarity ranking (no color data available)

1105 ranked_poems = candidates

1106 end

1108 -- Issue 8-050c: Get word's semantic color for header

1109 local word_color_entry = word_colors[word]

1110 local word_semantic_color = word_color_entry and word_color_entry.color or "gray"

1111 local word_hex_color = color_config and color_config[word_semantic_color] or "#888888"

1113 -- Issue 8-050e: Compute centroid-based chronological link

1114 local chrono_center_link = nil

1115 do

1116 -- Compute centroid of selected poems

1117 local centroid = compute_centroid(ranked_poems, poem_lookup)

1118 if centroid then

1119 -- Find the poem closest to the centroid

1120 local center_poem = find_closest_poem_to_centroid(centroid, poem_lookup, poems_by_index)

1121 if center_poem and center_poem.poem_index then

1122 -- Anchor must match the spans the chronological pages emit:

1123 -- get_poem_anchor_id() = "poem-<poem_index>". The old

1124 -- "poem-CATEGORY-ID" form matched no anchor, so this top

1125 -- "chronological" link landed at the page top, not the poem.

1126 local anchor_id = string.format("poem-%d", center_poem.poem_index)

1127 -- Get chronological page for this poem

1128 -- Issue 10-036: Use "01" fallback instead of "index" (redirect loses anchors)

1129 local chrono_page = chrono_page_map[center_poem.poem_index] or "01"

1130 -- Build full link

1131 local base_path = ".."

1132 chrono_center_link = string.format("%s/chronological/%s.html#%s",

1133 base_path, chrono_page, anchor_id)

1134 end

1135 end

1136 end

1138 -- Generate page with semantic colors and chronological position

1139 -- Issue 10-036: Pass chrono_page_map for correct per-poem pagination links

1140 if generate_word_page(word, ranked_poems, output_dir, CONFIG.poems_per_word_page, poem_colors, color_config, chrono_map, word_hex_color, chrono_center_link, chrono_page_map) then

1141 pages_generated = pages_generated + 1

1142 end

1143 else

1144 utils.log_warn(string.format("Missing embedding for word '%s', skipping", word))

1145 end

1146 end

1147 -- Close the animated line. progress.finish() is a no-op in verbose and quiet

1148 -- modes, unlike the bare print("") this replaced, which emitted a stray blank

1149 -- line into every piped log.

1150 progress.finish()

1152 utils.log_info(string.format("Generated %d word similarity pages in %s/wordcloud/", pages_generated, output_dir))

1153 return pages_generated

1159function M.generate_word_pages(options)

1160 options = options or {}

1162 -- Stage 1: Generate embeddings

1163 local embed_count = M.generate_word_embeddings(options)

1164 if not embed_count then

1165 return nil

1166 end

1168 -- Stage 2: Generate HTML

1169 return M.generate_word_html(options)

1174function M.main(mode)

1175 mode = mode or RUN_MODE

1177 if mode == "embeddings" then

1178 return M.generate_word_embeddings()

1179 elseif mode == "html" then

1180 return M.generate_word_html()

1181 else

1182 return M.generate_word_pages()

1183 end

1188if arg and #arg >= 0 and debug.getinfo(3) == nil then

1189 if RUN_MODE == "help" then

1190 print("Usage: luajit src/generate-word-pages.lua [DIR] [OPTIONS]")

1191 print("")

1192 print("Generates similarity pages for word cloud words.")

1193 print("For each word, creates a page showing poems ranked by semantic similarity.")

1194 print("")

1195 print("Options:")

1196 print(" DIR Project directory (default: /mnt/mtwo/programming/ai-stuff/neocities-modernization)")

1197 print(" --embeddings-only Generate word embeddings only (Stage 6 - expensive)")

1198 print(" --html-only Generate HTML pages only (Stage 9 - fast, requires embeddings)")

1199 print(" --all Include all words (no max_words limit)")

1200 print(" --words N Set maximum words to process (default: 200 from config)")

1201 print(" --help Show this help message")

1202 print("")

1203 print("Pipeline Integration (Issue 8-043b):")

1204 print(" Stage 6 (Embeddings): luajit src/generate-word-pages.lua --embeddings-only")

1205 print(" Stage 9 (HTML): luajit src/generate-word-pages.lua --html-only")

1206 print("")

1207 print("Without flags, runs both stages (backward compatible).")

1208 os.exit(0)

1209 end

1211 M.main()