scripts/precompute-diversity-sequences

#!/usr/bin/env luajit

-- Pre-computes diversity sequences for all poems and caches to disk
-- Uses centroid-based O(n^2) algorithm with thermal management (sleep between batches)
-- Output: assets/embeddings/embeddinggemma_latest/diversity_cache.json
--
-- OPTIMIZATIONS (Issue 9-003):
-- 1. Incremental running sum: O(768) per iteration instead of O(N × 768)
-- - Keeps a running sum instead of recalculating centroid from scratch
-- - ~4,000× faster centroid maintenance
--
-- 2. No division needed: cosine distance is scale-invariant
-- - cos(kA, B) = cos(A, B) for any scalar k
-- - Running sum gives same ranking as running average
--
-- 3. RAM-only storage: no temp files, sequences stored in memory
-- - Reduces disk I/O and SSD wear
-- - Results returned directly from worker threads via effil

-- {{{ local function setup_dir_path
local function setup_dir_path(provided_dir)
if provided_dir then
return provided_dir
end
return "/mnt/mtwo/programming/ai-stuff/neocities-modernization"
end
-- }}}

local DIR = setup_dir_path(arg and arg[1])

-- Add effil library path
package.cpath = "/home/ritz/programming/ai-stuff/libs/lua/effil-jit/build/?.so;" .. package.cpath
package.path = DIR .. "/libs/?.lua;" .. DIR .. "/src/?.lua;" .. package.path

local effil = require("effil")
local utils = require("utils")
local dkjson = require("dkjson")

-- {{{ local function relative_path
-- Issue 7-003: Show project name instead of "./" when path equals DIR
local function relative_path(absolute_path)
if absolute_path == DIR or absolute_path == DIR .. "/" then
local dir_name = DIR:match("([^/]+)/?$")
return dir_name .. "/"
end
if absolute_path:sub(1, #DIR) == DIR then
local rel = absolute_path:sub(#DIR + 1)
if rel:sub(1, 1) == "/" then rel = rel:sub(2) end
return "./" .. rel
end
return absolute_path
end
-- }}}

-- Configuration
-- Issue 9-003 Phase B: Increased default threads, reduced thermal sleep
-- Issue 8-025: Also check DIVERSITY_THREADS env var (set by run.sh)
-- Issue 8-027: Parse named flags for pagination settings

local NUM_THREADS = tonumber(os.getenv("DIVERSITY_THREADS")) or 8
local SLEEP_BETWEEN_BATCHES = 1 -- Was 5s, reduced to 1s
local TEST_MODE = false
local MAX_TEST_SEQUENCES = 10
local PAGES = nil
local POEMS_PER_PAGE = nil
local SEQUENCE_LIMIT = nil -- Issue 8-027: Direct sequence limit override
local FORCE_REGENERATE = false -- Issue 8-027: Force full regeneration

-- Parse command line arguments
for i = 2, #arg do
local a = arg[i]
if a:match("^--threads=") then
NUM_THREADS = tonumber(a:match("^--threads=(%d+)")) or NUM_THREADS
elseif a:match("^--sleep=") then
SLEEP_BETWEEN_BATCHES = tonumber(a:match("^--sleep=(%d+)")) or SLEEP_BETWEEN_BATCHES
elseif a:match("^--pages=") then
PAGES = tonumber(a:match("^--pages=(%d+)"))
elseif a:match("^--poems%-per%-page=") then
POEMS_PER_PAGE = tonumber(a:match("^--poems%-per%-page=(%d+)"))
elseif a:match("^--sequence%-limit=") then
-- Issue 8-027: Direct sequence limit flag (highest precedence)
SEQUENCE_LIMIT = tonumber(a:match("^--sequence%-limit=(%d+)"))
elseif a == "--force" then
-- Issue 8-027: Force full regeneration (ignores existing cache)
FORCE_REGENERATE = true
elseif a == "--test" then
TEST_MODE = true
end
end

-- MAX_SEQUENCE_LENGTH: Only compute top N most different poems per sequence
-- Issue 8-027: Precedence: --sequence-limit > --pages×--poems-per-page > default(1500)
-- Default: 15 pages × 100 poems = 1500 (Issue 8-020 storage constraint)
local MAX_SEQUENCE_LENGTH
if SEQUENCE_LIMIT then
-- Direct override has highest precedence
MAX_SEQUENCE_LENGTH = SEQUENCE_LIMIT
elseif PAGES and POEMS_PER_PAGE then
MAX_SEQUENCE_LENGTH = PAGES * POEMS_PER_PAGE
elseif PAGES then
MAX_SEQUENCE_LENGTH = PAGES * 100 -- Default 100 poems per page
elseif POEMS_PER_PAGE then
MAX_SEQUENCE_LENGTH = 15 * POEMS_PER_PAGE -- Default 15 pages
else
MAX_SEQUENCE_LENGTH = 1500 -- Default: 15 × 100
end

-- {{{ Worker function for diversity sequence computation
-- Computes full diversity sequence using INCREMENTAL CENTROID optimization
-- Returns sequence directly (no temp files) - RAM-only storage
--
-- OPTIMIZATION NOTES (Issue 9-003):
-- 1. Incremental running sum instead of recalculating centroid from scratch
-- - Old: O(N × 768) per iteration where N grows to 7,792
-- - New: O(768) per iteration (constant time)
-- - Speedup: ~4,000× for centroid maintenance
--
-- 2. No division needed: cosine distance is scale-invariant
-- - cos(kA, B) = cos(A, B) for any scalar k
-- - Running sum gives same ranking as running average
--
-- 3. RAM-only: return sequence directly, no temp file I/O
--
-- 4. Shared effil.table: main thread parses JSON once, shares via effil.table
-- - Human-readable JSON remains the source of truth
-- - Zero parsing overhead per thread (vs ~60s JSON parsing each)
-- - Direct array access from shared memory
--
-- Issue 8-027: Added existing_sequence parameter for extension mode
-- When existing_sequence is provided, reconstruct running_sum and continue from there
local function diversity_sequence_worker(poem_id, poem_index, shared_embeddings, embedding_dim,
num_poems, max_sequence_length, progress_table, thread_idx,
existing_sequence)
-- {{{ Access embeddings from shared effil.table (FAST!)
-- The main thread parsed JSON once and converted to a flat effil.table
-- Workers access shared memory directly - no parsing, no file I/O
-- All embeddings are stored as: shared_embeddings[1..total_floats]
-- where total_floats = num_poems × embedding_dim
--
-- Issue 9-003 Phase B: Shared effil.table replaces per-thread JSON parsing
-- JSON remains the human-readable source of truth

-- Access shared embeddings directly - effil.table supports numeric indexing
local all_embeddings_flat = shared_embeddings
-- }}}

-- {{{ Helper: cosine distance between embedding at index and running sum
-- OPTIMIZED (Issue 9-003 Phase B): Direct array access, no table creation
-- emb_start: starting index in all_embeddings_flat (1-indexed)
-- running_sum: the accumulated centroid sum
-- Returns: cosine distance (1 - cosine_similarity)
local function cosine_distance_direct(emb_start, running_sum)
local dot_product = 0
local norm1 = 0
local norm2 = 0

-- Direct array access avoids table creation overhead
-- LuaJIT can optimize this tight loop well
for i = 1, embedding_dim do
local a = running_sum[i]
local b = all_embeddings_flat[emb_start + i - 1]
dot_product = dot_product + a * b
norm1 = norm1 + a * a
norm2 = norm2 + b * b
end

-- Fast inverse square root approximation not needed - sqrt is fast enough
norm1 = math.sqrt(norm1)
norm2 = math.sqrt(norm2)

if norm1 == 0 or norm2 == 0 then
return 1.0
end

return 1.0 - (dot_product / (norm1 * norm2))
end
-- }}}

-- {{{ Helper: add embedding at index to running sum
-- OPTIMIZED (Issue 9-003 Phase B): Direct array access
local function add_to_running_sum(emb_start, running_sum)
for i = 1, embedding_dim do
running_sum[i] = running_sum[i] + all_embeddings_flat[emb_start + i - 1]
end
end
-- }}}

-- {{{ Helper: compute embedding start index
-- Inline this calculation to avoid function call overhead
local function emb_start_for(idx)
return (idx - 1) * embedding_dim + 1
end
-- }}}

-- Issue 8-027: Extension mode support
-- If existing_sequence is provided, reconstruct running_sum and continue from there
local sequence
local running_sum = {}
local remaining = {}
local target_length = max_sequence_length or MAX_SEQUENCE_LENGTH

if existing_sequence and #existing_sequence > 0 then
-- EXTENSION MODE: Continue from existing sequence
-- 1. Copy existing sequence as our starting point
sequence = {}
for i = 1, #existing_sequence do
sequence[i] = existing_sequence[i]
end

-- 2. Reconstruct running_sum by summing embeddings of all poems in sequence
-- This is O(N × 768) where N = existing sequence length
for i = 1, embedding_dim do
running_sum[i] = 0
end
for _, idx in ipairs(sequence) do
local emb_start = emb_start_for(idx)
for i = 1, embedding_dim do
running_sum[i] = running_sum[i] + all_embeddings_flat[emb_start + i - 1]
end
end

-- 3. Build remaining = all_poems - sequence (poems not yet selected)
local in_sequence = {}
for _, idx in ipairs(sequence) do
in_sequence[idx] = true
end
for i = 1, num_poems do
if not in_sequence[i] then
table.insert(remaining, {idx = i, emb_start = emb_start_for(i)})
end
end

-- Issue 8-026: Update progress to reflect starting point
if progress_table and thread_idx then
progress_table[thread_idx] = #sequence
end
else
-- FRESH COMPUTATION: Build from scratch (original behavior)
-- Build starting embedding and initialize running sum
-- OPTIMIZED (Issue 9-003 Phase B): Direct flat array access
local start_emb_idx = emb_start_for(poem_index)
for i = 1, embedding_dim do
running_sum[i] = all_embeddings_flat[start_emb_idx + i - 1]
end

-- Build list of other poem indices with pre-computed embedding starts
-- OPTIMIZED: Store emb_start alongside index to avoid recomputation
for i = 1, num_poems do
if i ~= poem_index then
table.insert(remaining, {idx = i, emb_start = emb_start_for(i)})
end
end

-- Start with this poem's index
sequence = {poem_index}
end

while #remaining > 0 and #sequence < target_length do
-- Find poem with maximum distance from running_sum (acts as centroid)
-- No need to divide - cosine is scale-invariant, ranking is preserved
local max_dist = -1
local max_remaining_idx = -1

-- OPTIMIZED: Use direct array access via cosine_distance_direct
for i = 1, #remaining do
local r = remaining[i]
local dist = cosine_distance_direct(r.emb_start, running_sum)
if dist > max_dist then
max_dist = dist
max_remaining_idx = i
end
end

if max_remaining_idx > 0 then
local selected = remaining[max_remaining_idx]
table.insert(sequence, selected.idx)

-- OPTIMIZED: Update running sum directly from flat array O(768)
add_to_running_sum(selected.emb_start, running_sum)

-- Remove selected from remaining (swap-and-pop for O(1) removal)
-- OPTIMIZED: Avoid table.remove which is O(n)
local last = remaining[#remaining]
remaining[max_remaining_idx] = last
remaining[#remaining] = nil

-- Issue 8-026: Update progress every iteration (effil.table writes are fast)
if progress_table and thread_idx then
progress_table[thread_idx] = #sequence
end
else
break
end
end

-- Issue 8-026: Mark complete
if progress_table and thread_idx then
progress_table[thread_idx] = target_length
end

-- Return sequence directly (RAM-only, no temp file)
return sequence
end
-- }}}

-- {{{ Main execution
print("=" .. string.rep("=", 60))
print("Diversity Sequence Pre-computation")
print("=" .. string.rep("=", 60))
print(string.format("Project directory: %s", relative_path(DIR)))
print(string.format("Thread count: %d", NUM_THREADS))
print(string.format("Sleep between batches: %d seconds", SLEEP_BETWEEN_BATCHES))
print(string.format("Test mode: %s", TEST_MODE and "yes (first " .. MAX_TEST_SEQUENCES .. " sequences)" or "no"))

-- {{{ Pipeline validation (Issue 10-011 Phase G)
print("\n🔍 Validating pipeline data...")
local validator = require("pipeline-validator")
validator.set_verbose(false)

-- Check embeddings and similarity matrix (both required for diversity computation)
local emb = validator.check_embeddings("embeddinggemma_latest")
local sim = validator.check_similarity_matrix("embeddinggemma_latest")

if not emb.complete or not sim.complete then
print("\n" .. string.rep("=", 60))
io.stderr:write("\n")
io.stderr:write("❌ ERROR: Pipeline validation failed:\n")

if not emb.complete then
io.stderr:write(string.format(" • Embeddings incomplete: %d/%d poems\n", emb.count, emb.total))
io.stderr:write(" Fix: ./generate-embeddings.sh --incremental\n")
end

if not sim.complete then
io.stderr:write(string.format(" • Similarity matrix incomplete: %d/%d poems\n", sim.count, sim.total))
io.stderr:write(" Fix: lua src/similarity-engine-parallel.lua\n")
end

io.stderr:write("\nCannot compute diversity sequences without complete embeddings\n")
io.stderr:write("and similarity matrix. Run the commands above to fix.\n\n")
os.exit(1)
end

-- Check freshness (warning only)
local fresh = validator.check_freshness("embeddinggemma_latest")
if not fresh.fresh then
io.stderr:write("\n⚠️ WARNING: Pipeline data may be stale:\n")
for _, issue in ipairs(fresh.issues) do
io.stderr:write(string.format(" • %s\n", issue.message))
io.stderr:write(string.format(" Fix: %s\n", issue.fix))
end
io.stderr:write("\nContinuing anyway. Results may not reflect latest poems.\n\n")
end

print(" ✅ Pipeline validation passed")
-- }}}

-- Load data files
print("\n🔄 Loading data files (single-threaded phase)...")
local poems_file = DIR .. "/assets/poems.json"
local embeddings_file = DIR .. "/assets/embeddings/embeddinggemma_latest/embeddings.json"

io.write(" Loading poems.json... ")
io.flush()
local poems_data = utils.read_json_file(poems_file)
print("done")

if not poems_data then
print("❌ Error: Could not load poems.json")
os.exit(1)
end

-- Embedding validation: full check in production, lightweight in test mode
local embedding_dim = 768 -- Default dimension
local embeddings_data = nil
local missing_embeddings = {}
local random_embeddings = 0

if TEST_MODE then
-- Lightweight check: just verify file exists and is readable
print(" Checking embeddings.json (lightweight, test mode)...")
print(" [DEBUG] Step 1: opening file")
local f = io.open(embeddings_file, "r")
print(" [DEBUG] Step 2: file handle = " .. tostring(f))
if f then
print(" [DEBUG] Step 3: reading")
local first_char = f:read(1)
print(" [DEBUG] Step 4: char = " .. tostring(first_char))
f:close()
if first_char == "{" then
print(" ✓ File OK")
else
print("❌ Error: embeddings.json appears malformed")
os.exit(1)
end
else
print("❌ Error: Could not open embeddings.json")
os.exit(1)
end
print(" ⚡ Skipping full validation in test mode")
else
-- Full validation in production mode
io.write(" Loading embeddings.json (62MB, ~1-2 minutes)... ")
io.flush()
embeddings_data = utils.read_json_file(embeddings_file)
print("done")

if not embeddings_data then
print("❌ Error: Could not load embeddings.json")
os.exit(1)
end

embedding_dim = embeddings_data.metadata and embeddings_data.metadata.embedding_dimension or 768
local invalid_dimensions = {}

for i, poem in ipairs(poems_data.poems) do
if poem.id and poem.content and poem.content ~= "" then
local emb_entry = embeddings_data.embeddings[i]
if not emb_entry or not emb_entry.embedding then
table.insert(missing_embeddings, poem.id)
elseif type(emb_entry.embedding) ~= "table" or #emb_entry.embedding ~= embedding_dim then
table.insert(invalid_dimensions, poem.id)
elseif emb_entry.is_random then
random_embeddings = random_embeddings + 1
end
end
end

if #missing_embeddings > 0 or #invalid_dimensions > 0 then
print("\n❌ EMBEDDING VALIDATION FAILED")
print("════════════════════════════════════════════════════════════════════════════")
if #missing_embeddings > 0 then
print(string.format(" Missing embeddings: %d poems with content have no embedding", #missing_embeddings))
print(" First 10 IDs: " .. table.concat({unpack(missing_embeddings, 1, 10)}, ", "))
end
if #invalid_dimensions > 0 then
print(string.format(" Invalid dimensions: %d poems have wrong embedding size", #invalid_dimensions))
print(" First 10 IDs: " .. table.concat({unpack(invalid_dimensions, 1, 10)}, ", "))
end
print("")
print(" Please regenerate embeddings before pre-computing diversity:")
print(" lua src/similarity-engine.lua -I (option 1)")
print("════════════════════════════════════════════════════════════════════════════")
os.exit(1)
end

print("✅ Data files loaded")
print(string.format(" 📄 Poems: %d", #poems_data.poems))
print(string.format(" 🧮 Embeddings: %d (dim=%d)", #embeddings_data.embeddings, embedding_dim))
if random_embeddings > 0 then
print(string.format(" ℹ️ %d empty poems have random embeddings (expected)", random_embeddings))
end

end

-- {{{ Create shared effil.table for worker threads (Issue 9-003 Phase B)
-- JSON remains the human-readable source of truth - only parsed once here
-- Workers access shared memory directly - no parsing overhead per thread
-- Flat array format: embeddings[1..total_floats] where total_floats = num_poems × embedding_dim
--
-- NOTE: In test mode, we need to load embeddings for the shared table
local shared_embeddings = nil

if TEST_MODE then
-- In test mode, we skipped loading embeddings earlier - load now for shared table
print("\n📦 Loading embeddings.json for shared memory (test mode)...")
embeddings_data = utils.read_json_file(embeddings_file)
if not embeddings_data then
print("❌ Error: Could not load embeddings.json")
os.exit(1)
end
embedding_dim = embeddings_data.metadata and embeddings_data.metadata.embedding_dimension or 768
end

if embeddings_data then
print("📦 Creating shared effil.table for worker threads...")

local num_emb = #embeddings_data.embeddings
local total_floats = num_emb * embedding_dim

-- Build effil.table incrementally (avoids hanging on large constructor calls)
-- effil.table() with no args creates an empty table we can populate
shared_embeddings = effil.table()

-- Populate in batches with progress reporting
local BATCH_SIZE = 100000 -- 100k floats per batch
local idx = 1
local last_percent = -1

for i = 1, num_emb do
local emb = embeddings_data.embeddings[i]
if emb and emb.embedding then
for j = 1, embedding_dim do
shared_embeddings[idx] = emb.embedding[j] or 0
idx = idx + 1
end
else
for _ = 1, embedding_dim do
shared_embeddings[idx] = 0
idx = idx + 1
end
end

-- Progress reporting every 1%
local percent = math.floor(i * 100 / num_emb)
if percent > last_percent and percent % 5 == 0 then
io.write(string.format("\r Building shared effil.table: %d%% (%d/%d poems)",
percent, i, num_emb))
io.flush()
last_percent = percent
end
end
print(string.format("\r Building shared effil.table: 100%% (%d poems) ", num_emb))

print(string.format(" ✅ Shared effil.table ready: %d floats (%d poems × %d dim)",
total_floats, num_emb, embedding_dim))
end
-- }}}

-- Issue 8-025: Build poem index mappings BEFORE releasing embeddings_data
-- This loop must run while embeddings_data is still available for validation
local poem_id_to_index = {}
local index_to_poem_id = {}
local valid_poem_ids = {}
local num_poems = #poems_data.poems -- Total poems (for worker indexing)

for i, poem in ipairs(poems_data.poems) do
if poem.id then
if TEST_MODE then
-- In test mode, assume all poems are valid (workers will handle errors)
poem_id_to_index[poem.id] = i
index_to_poem_id[i] = poem.id
table.insert(valid_poem_ids, poem.id)
elseif embeddings_data and embeddings_data.embeddings[i] and embeddings_data.embeddings[i].embedding then
-- In production mode, only include poems with valid embeddings
poem_id_to_index[poem.id] = i
index_to_poem_id[i] = poem.id
table.insert(valid_poem_ids, poem.id)
end
end
end

table.sort(valid_poem_ids)

-- Limit in test mode
if TEST_MODE then
local limited_ids = {}
for i = 1, math.min(MAX_TEST_SEQUENCES, #valid_poem_ids) do
table.insert(limited_ids, valid_poem_ids[i])
end
valid_poem_ids = limited_ids
end

-- Issue 8-026: Load existing cache for incremental resume
-- Issue 8-027: Detect extend/truncate scenarios based on sequence_limit
local cache_file = DIR .. "/assets/embeddings/embeddinggemma_latest/diversity_cache.json"
local existing_sequences = {}
local existing_count = 0
local existing_sequence_limit = 0
local existing_embeddings_size = 0
local needs_extension = false
local sequences_to_extend = {} -- poem_id -> current_length (for extension)

local cache_handle = io.open(cache_file, "r")
if cache_handle and not FORCE_REGENERATE then
print(" 📂 Found existing cache, loading for incremental resume...")
local cache_content = cache_handle:read("*all")
cache_handle:close()

local existing_cache = dkjson.decode(cache_content)
if existing_cache and existing_cache.sequences then
-- Issue 8-027: Extract existing metadata for comparison
if existing_cache.metadata then
existing_sequence_limit = existing_cache.metadata.sequence_limit or 0
existing_embeddings_size = existing_cache.metadata.embeddings_file_size or 0
end

-- Issue 8-027: Check for embeddings file size change (staleness warning)
local current_emb_size = 0
local size_handle = io.popen(string.format("stat -c %%s '%s' 2>/dev/null", embeddings_file))
if size_handle then
local size_str = size_handle:read("*a")
size_handle:close()
current_emb_size = tonumber(size_str:match("%d+")) or 0
end
if existing_embeddings_size > 0 and current_emb_size ~= existing_embeddings_size then
print(string.format(" ⚠️ WARNING: embeddings.json size changed (%d → %d bytes)",
existing_embeddings_size, current_emb_size))
print(" ⚠️ Cache may be stale. Use --force to regenerate completely.")
end

-- Issue 8-027: Determine operation mode based on sequence_limit comparison
if existing_sequence_limit > 0 then
if MAX_SEQUENCE_LENGTH < existing_sequence_limit then
-- TRUNCATION: requested < existing → instant slice operation
print(string.format(" ✂️ TRUNCATION mode: %d → %d (slicing arrays, no computation)",
existing_sequence_limit, MAX_SEQUENCE_LENGTH))

local truncated_count = 0
for poem_id_str, sequence in pairs(existing_cache.sequences) do
if #sequence > MAX_SEQUENCE_LENGTH then
-- Slice sequence to requested length
local truncated = {}
for i = 1, MAX_SEQUENCE_LENGTH do
truncated[i] = sequence[i]
end
existing_sequences[poem_id_str] = truncated
truncated_count = truncated_count + 1
else
existing_sequences[poem_id_str] = sequence
end
existing_count = existing_count + 1
end

print(string.format(" ✅ Truncated %d sequences to length %d",
truncated_count, MAX_SEQUENCE_LENGTH))

-- For truncation, we're done - skip all computation
valid_poem_ids = {} -- Nothing to compute

elseif MAX_SEQUENCE_LENGTH > existing_sequence_limit then
-- EXTENSION: requested > existing → need to continue algorithm
print(string.format(" 📈 EXTENSION mode: %d → %d (continuing algorithm)",
existing_sequence_limit, MAX_SEQUENCE_LENGTH))
needs_extension = true

-- Load all existing sequences, mark those needing extension
for poem_id_str, sequence in pairs(existing_cache.sequences) do
existing_sequences[poem_id_str] = sequence
existing_count = existing_count + 1

-- If sequence is shorter than requested, mark for extension
if #sequence < MAX_SEQUENCE_LENGTH then
local poem_id = tonumber(poem_id_str)
if poem_id then
sequences_to_extend[poem_id] = #sequence
end
end
end

-- valid_poem_ids = only poems that need extension
local extension_ids = {}
for poem_id, _ in pairs(sequences_to_extend) do
table.insert(extension_ids, poem_id)
end
table.sort(extension_ids)
valid_poem_ids = extension_ids

print(string.format(" ✅ Loaded %d existing sequences, %d need extension to %d",
existing_count, #valid_poem_ids, MAX_SEQUENCE_LENGTH))

else
-- SAME LIMIT: requested == existing → standard incremental resume
print(string.format(" ✓ Same sequence limit (%d), standard incremental mode",
MAX_SEQUENCE_LENGTH))

-- Copy existing sequences and build skip set
local skip_ids = {}
for poem_id_str, sequence in pairs(existing_cache.sequences) do
existing_sequences[poem_id_str] = sequence
skip_ids[tonumber(poem_id_str)] = true
existing_count = existing_count + 1
end

-- Filter out already-computed poem IDs
local remaining_ids = {}
for _, poem_id in ipairs(valid_poem_ids) do
if not skip_ids[poem_id] then
table.insert(remaining_ids, poem_id)
end
end
valid_poem_ids = remaining_ids

print(string.format(" ✅ Loaded %d existing sequences, %d remaining to compute",
existing_count, #valid_poem_ids))
end
else
-- No sequence_limit in metadata (old cache format) → treat as standard resume
print(" ℹ️ Legacy cache format (no sequence_limit), standard incremental mode")
local skip_ids = {}
for poem_id_str, sequence in pairs(existing_cache.sequences) do
existing_sequences[poem_id_str] = sequence
skip_ids[tonumber(poem_id_str)] = true
existing_count = existing_count + 1
end

local remaining_ids = {}
for _, poem_id in ipairs(valid_poem_ids) do
if not skip_ids[poem_id] then
table.insert(remaining_ids, poem_id)
end
end
valid_poem_ids = remaining_ids

print(string.format(" ✅ Loaded %d existing sequences, %d remaining to compute",
existing_count, #valid_poem_ids))
end
end
elseif FORCE_REGENERATE then
print(" 🔄 --force specified, ignoring existing cache")
else
print(" 📂 No existing cache found, starting fresh")
end

-- Now release embeddings_data - effil.table owns the data, valid_poem_ids is built
embeddings_data = nil
collectgarbage("collect")
print(" 🗑️ Released source data (effil.table now owns memory)")

print(string.format(" 🔢 Sequences to compute: %d", #valid_poem_ids))

-- RAM-only storage: no temp files needed (Issue 9-003 optimization)
-- Sequences are returned directly from worker threads and stored in memory
local computed_sequences = {} -- poem_id -> sequence (array of indices)

-- Issue 8-027: Initialize timing and counters early (needed for conditional flow)
local start_time = os.time()
local completed = 0
local failed = 0
local skip_computation = false

-- Issue 8-027: Early exit for truncation mode (no computation needed)
-- Skip all expensive operations and go straight to saving
if #valid_poem_ids == 0 and existing_count > 0 then
print("\n✅ No computation needed (truncation mode or all sequences exist)")
print(" Skipping embedding loading and thread pool setup...")
skip_computation = true
end

if not skip_computation then
-- BEGIN COMPUTATION BLOCK (skipped for truncation mode)

-- Verify shared embeddings are ready
if not shared_embeddings then
print("❌ Error: shared_embeddings not initialized")
os.exit(1)
end
print("\n✅ Shared effil.table ready for worker threads")

-- Estimate time (with all optimizations from Issue 9-003)
-- Original O(N³): 8000 poems × 8000 iterations × 768 dim = 49 trillion ops → ~42 hours
-- With sequence limit: 8000 × 1500 iterations = ~5× faster → ~8-10 hours
-- With 8 threads + other optimizations: ~4-6 hours total
local est_time_per_seq = 4 -- seconds (with 1500 limit + optimizations)
local est_total_time = (#valid_poem_ids * est_time_per_seq) / NUM_THREADS
local est_hours = est_total_time / 3600
print(string.format("\n⏱️ Estimated time: %.1f hours (%.0f seconds per sequence)", est_hours, est_time_per_seq))
local pages_used = PAGES or 15
local poems_per_page_used = POEMS_PER_PAGE or 100
print(string.format(" Sequence limit: %d poems (%d pages × %d poems/page)",
MAX_SEQUENCE_LENGTH, pages_used, poems_per_page_used))
print(string.format(" With %d-second sleep every %d sequences", SLEEP_BETWEEN_BATCHES, NUM_THREADS))
print(" 📁 Progress saved after each batch (Ctrl+C safe)")

-- Pre-computation (MULTI-THREADED phase)
print("\n🚀 Starting diversity sequence pre-computation (MULTI-THREADED)...")
print(string.format(" Using %d threads with %d-second cooling between batches", NUM_THREADS, SLEEP_BETWEEN_BATCHES))
print(" Note: This is a one-time cost. Results will be cached.")
-- Issue 8-027: start_time, completed, failed defined earlier (before goto)
start_time = os.time() -- Reset start time to actual computation start
local batch_num = 0

-- Process in batches with thermal management
local batch_start = 1
while batch_start <= #valid_poem_ids do
batch_num = batch_num + 1
local batch_end = math.min(batch_start + NUM_THREADS - 1, #valid_poem_ids)
local batch_size = batch_end - batch_start + 1

print(string.format("\n Batch %d: sequences %d-%d", batch_num, batch_start, batch_end))

-- Issue 8-026: Create shared progress table for thread visualization
local progress_table = effil.table()
for i = 1, batch_size do
progress_table[i] = 0
end

-- Start all threads in this batch simultaneously
local threads = {}
local thread_poem_ids = {} -- Track which poem ID each thread is processing
local batch_start_time = os.time()

for i = batch_start, batch_end do
local poem_id = valid_poem_ids[i]
local poem_index = poem_id_to_index[poem_id]
local thread_idx = i - batch_start + 1

-- Issue 8-027: For extension mode, get existing sequence and convert to indices
local existing_seq_indices = nil
if needs_extension and existing_sequences[tostring(poem_id)] then
local existing_seq = existing_sequences[tostring(poem_id)]
existing_seq_indices = effil.table() -- Must use effil.table for thread safety
for j, pid in ipairs(existing_seq) do
local idx = poem_id_to_index[pid]
if idx then
existing_seq_indices[j] = idx
end
end
end

thread_poem_ids[thread_idx] = poem_id
threads[thread_idx] = effil.thread(diversity_sequence_worker)(
poem_id,
poem_index,
shared_embeddings, -- Shared effil.table (JSON parsed once, instant access)
embedding_dim,
num_poems,
MAX_SEQUENCE_LENGTH, -- Limit sequence to 1,500 poems (Issue 8-020)
progress_table, -- Issue 8-026: Shared progress table
thread_idx, -- Issue 8-026: Thread index for progress slot
existing_seq_indices -- Issue 8-027: Existing sequence for extension mode
)
end

-- Issue 8-026: Poll threads with progress display instead of blocking get()
local threads_done = {}
local all_done = false

while not all_done do
all_done = true

-- Check each thread's status
for i = 1, batch_size do
if not threads_done[i] then
-- wait(0) = non-blocking check
local status = threads[i]:wait(0)
if status == "completed" or status == true then
threads_done[i] = true
else
all_done = false
end
end
end

-- Display progress line
-- Issue 8-026: Use A, B, C... for thread names, clear line before writing
-- Each thread gets a unique color for easy visual tracking
local thread_letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
local thread_colors = {
"\27[91m", -- A: bright red
"\27[92m", -- B: bright green
"\27[93m", -- C: bright yellow
"\27[94m", -- D: bright blue
"\27[95m", -- E: bright magenta
"\27[96m", -- F: bright cyan
"\27[31m", -- G: red
"\27[32m", -- H: green
"\27[33m", -- I: yellow
"\27[34m", -- J: blue
"\27[35m", -- K: magenta
"\27[36m", -- L: cyan
"\27[97m", -- M: bright white
"\27[90m", -- N: bright black (gray)
"\27[91m", -- O: bright red (cycle)
"\27[92m", -- P: bright green (cycle)
}
local reset_color = "\27[0m"

local progress_parts = {}
local total_progress = 0
for i = 1, batch_size do
local p = tonumber(progress_table[i]) or 0
total_progress = total_progress + p
local letter = thread_letters:sub(i, i)
local color = thread_colors[((i - 1) % #thread_colors) + 1]
if threads_done[i] then
table.insert(progress_parts, string.format("%s%s:✓%s", color, letter, reset_color))
else
table.insert(progress_parts, string.format("%s%s:%d%s", color, letter, p, reset_color))
end
end

local avg_progress = math.floor(total_progress / batch_size)
local elapsed = os.time() - batch_start_time
local bar_width = 20
local filled = math.floor((avg_progress / MAX_SEQUENCE_LENGTH) * bar_width)
local bar = string.rep("█", filled) .. string.rep("░", bar_width - filled)

-- Clear current line using ANSI escape code, then write new content
-- \27[2K = erase entire line, \r = carriage return to start
io.write("\27[2K\r")
io.write(string.format(" [%s] %d/%d avg | %s | %ds",
bar, avg_progress, MAX_SEQUENCE_LENGTH,
table.concat(progress_parts, " "), elapsed))
io.flush()

if not all_done then
-- Use effil.sleep instead of os.execute to allow Ctrl+C interruption
effil.sleep(0.5) -- effil.sleep takes seconds, not milliseconds
end
end

-- Clear the progress line
io.write("\27[2K\r")
io.flush()

-- Collect results from all threads
for i = 1, batch_size do
local sequence, err = threads[i]:get()
local poem_id = thread_poem_ids[i]

-- effil returns tables as effil.table - convert to regular Lua table
if sequence and type(sequence) == "userdata" then
-- Convert effil.table to regular Lua table
local lua_sequence = {}
for j = 1, #sequence do
lua_sequence[j] = sequence[j]
end
sequence = lua_sequence
end

if sequence and #sequence > 0 then
computed_sequences[poem_id] = sequence -- Store in RAM
completed = completed + 1
else
failed = failed + 1
print(string.format(" ⚠️ Failed: poem %d (error: %s)", poem_id, tostring(err or "empty sequence")))
end
end

local batch_elapsed = os.time() - batch_start_time
local total_elapsed = os.time() - start_time
local rate = completed / math.max(total_elapsed, 1)
local remaining = #valid_poem_ids - completed - failed
local eta_seconds = remaining / math.max(rate, 0.001)
local eta_hours = eta_seconds / 3600

print(string.format(" ✅ Batch complete: %d/%d total (%.2f seq/min, ETA: %.1f hours)",
completed, #valid_poem_ids, rate * 60, eta_hours))

-- Issue 8-027: Incremental save after each batch (Ctrl+C safe)
-- Merge newly computed sequences into existing_sequences and save to disk
local batch_save_count = 0
for poem_id, sequence_indices in pairs(computed_sequences) do
-- Convert indices to poem IDs
local sequence_ids = {}
for _, idx in ipairs(sequence_indices) do
local pid = index_to_poem_id[idx]
if pid then
table.insert(sequence_ids, pid)
end
end
existing_sequences[tostring(poem_id)] = sequence_ids
batch_save_count = batch_save_count + 1
end
computed_sequences = {} -- Clear after merging

-- Build and save cache
local save_cache = {
metadata = {
generated_at = os.date("%Y-%m-%d %H:%M:%S"),
total_sequences = existing_count + completed,
algorithm = "centroid-based-diversity-incremental",
algorithm_version = "centroid-v1",
sequence_limit = MAX_SEQUENCE_LENGTH,
min_sequence_length = MAX_SEQUENCE_LENGTH, -- Will be recalculated at end
embedding_dimension = embedding_dim,
source_embeddings = "embeddinggemma_latest",
optimization_notes = "Incremental running sum, no division, RAM-only storage"
},
sequences = existing_sequences
}
local save_json = dkjson.encode(save_cache, {indent = false})
local save_f = io.open(cache_file, "w")
if save_f then
save_f:write(save_json)
save_f:close()
io.write(string.format(" 💾 Saved %d sequences to cache\n", existing_count + completed))
io.flush()
end

-- Thermal management: sleep before next batch (unless this is the last batch)
-- Use effil.sleep to allow Ctrl+C interruption
if batch_end < #valid_poem_ids then
print(string.format(" 😴 Cooling down for %d seconds...", SLEEP_BETWEEN_BATCHES))
effil.sleep(SLEEP_BETWEEN_BATCHES) -- effil.sleep takes seconds
end

batch_start = batch_end + 1
end

local total_elapsed = os.time() - start_time
print(string.format("\n✅ Pre-computation complete: %d sequences in %d seconds (%.1f hours)",
completed, total_elapsed, total_elapsed / 3600))

if failed > 0 then
print(string.format("⚠️ %d sequences failed", failed))
end

end -- Issue 8-027: END COMPUTATION BLOCK (if not skip_computation)

-- Collect all sequences into final cache file (from RAM, no temp files)
print("\n📦 Building cache from RAM-stored sequences...")

-- Issue 8-026: Merge existing sequences with newly computed ones
local total_sequences = existing_count + completed

-- Issue 8-027: Get embeddings file size for staleness detection
local embeddings_file_size = 0
local size_handle = io.popen(string.format("stat -c %%s '%s' 2>/dev/null", embeddings_file))
if size_handle then
local size_str = size_handle:read("*a")
size_handle:close()
embeddings_file_size = tonumber(size_str:match("%d+")) or 0
end

-- Issue 8-027: Calculate min_sequence_length from all sequences
local min_sequence_length = MAX_SEQUENCE_LENGTH
for _, sequence in pairs(existing_sequences) do
if #sequence < min_sequence_length then
min_sequence_length = #sequence
end
end
for _, sequence in pairs(computed_sequences) do
if #sequence < min_sequence_length then
min_sequence_length = #sequence
end
end

local cache = {
metadata = {
generated_at = os.date("%Y-%m-%d %H:%M:%S"),
total_sequences = total_sequences,
algorithm = "centroid-based-diversity-incremental", -- Issue 9-003 optimized
algorithm_version = "centroid-v1", -- Issue 8-027: For cache invalidation
sequence_limit = MAX_SEQUENCE_LENGTH, -- Issue 8-027: Target limit used
min_sequence_length = min_sequence_length, -- Issue 8-027: Shortest sequence
embedding_dimension = embedding_dim,
source_embeddings = "embeddinggemma_latest",
embeddings_file_size = embeddings_file_size, -- Issue 8-027: For staleness detection
optimization_notes = "Incremental running sum, no division, RAM-only storage"
},
sequences = {}
}

-- First, copy existing sequences (already in poem ID format)
for poem_id_str, sequence in pairs(existing_sequences) do
cache.sequences[poem_id_str] = sequence
end

-- Convert NEW sequences from indices to poem IDs and add to cache
local newly_converted = 0
for poem_id, sequence_indices in pairs(computed_sequences) do
local sequence_ids = {}
for _, idx in ipairs(sequence_indices) do
local pid = index_to_poem_id[idx]
if pid then
table.insert(sequence_ids, pid)
end
end
cache.sequences[tostring(poem_id)] = sequence_ids
newly_converted = newly_converted + 1
end

print(string.format(" Merged %d existing + %d new = %d total sequences",
existing_count, newly_converted, total_sequences))

-- Write cache file (cache_file already defined above for incremental loading)
local cache_json = dkjson.encode(cache, {indent = false})
local f = io.open(cache_file, "w")
if f then
f:write(cache_json)
f:close()
print(string.format("✅ Cache file written: %s", relative_path(cache_file)))

-- Get file size
local handle = io.popen(string.format("ls -lh '%s' | awk '{print $5}'", cache_file))
if handle then
local size = handle:read("*a"):gsub("%s+", "")
handle:close()
print(string.format(" Size: %s", size))
end
else
print("❌ Error: Could not write cache file")
end

-- No temp file cleanup needed - RAM-only storage (Issue 9-003)

print("\n" .. string.rep("=", 61))
print("Pre-computation Summary")
print(string.rep("=", 61))
print(string.format(" Sequences computed: %d", completed))
print(string.format(" Failed: %d", failed))
print(string.format(" Total time: %d seconds (%.1f hours)", total_elapsed, total_elapsed / 3600))
print(string.format(" Cache file: %s", relative_path(cache_file)))
print("\nYou can now run generate-html-parallel which will use the cached sequences.")
-- }}}