src/similarity-engine.lua

4local function setup_dir_path(provided_dir)

5 if provided_dir then

6 return provided_dir

7 end

8 return "/mnt/mtwo/programming/ai-stuff/neocities-modernization"

33local embedding_models = {

34 -- Key is the GGUF-basename form ("nomic-embed-text-v1.5") to match what

35 -- config.lua, run.sh, and generate-embeddings.sh actually pass. The old

36 -- Ollama-era "model:tag" colon form ("nomic-embed-text:v1.5") never resolved

37 -- after the 10-049 migration, so a full regen aborted with "Unknown

38 -- embedding model" before sending any request. (Leftover from 10-049.)

39 ["nomic-embed-text-v1.5"] = {

40 dimensions = 768,

41 timeout = 30,

42 -- v1.5 routes through task-specific weights based on prompt prefix;

43 -- the active prefix is configured per inference_servers entry.

44 requires_prompt_prefix = true,

45 },

46 ["embeddinggemma:latest"] = {

47 dimensions = 768,

48 timeout = 30

49 },

50 -- Issue 10-031: GGUF-basename forms for the local model-comparison set.

51 ["mxbai-embed-large-v1"] = {

52 dimensions = 1024,

53 timeout = 30

54 },

55 ["embeddinggemma-300m"] = {

56 dimensions = 768,

57 timeout = 30,

58 -- Uses a clustering task prompt, configured per inference_servers entry.

59 requires_prompt_prefix = true,

60 },

61 ["qwen3-embedding:4b"] = {

62 dimensions = 2560,

63 timeout = 60 -- bigger model, longer per-call

64 },

65 ["qwen3-embedding:8b"] = {

66 dimensions = 4096,

67 timeout = 90

68 },

69 ["text-embedding-ada-002"] = {

70 dimensions = 1536,

71 timeout = 60

72 },

73 ["all-MiniLM-L6-v2"] = {

74 dimensions = 384,

75 timeout = 20

76 }

77}

81local function get_model_storage_path(base_dir, model_name)

82 -- Issue 10-054: the model's cache dir comes from embeddings_dir() so it

83 -- follows the RAM/disk switch -- this function is the EMBEDDING GENERATOR's

84 -- write path (similarity-engine.lua is the embedder behind generate-

85 -- embeddings.sh, not just legacy matrix code), so leaving it on disk is what

86 -- made the flip write embeddings where no reader looked. base_dir is now

87 -- ignored (kept in the signature for callers that still pass get_assets_root);

88 -- embeddings_dir builds the same <root>/embeddings/<safe_model> path and is

89 -- identical to the old base_dir/embeddings/<safe_model> while the switch is off.

90 local model_dir = utils.embeddings_dir(model_name)

92 -- Create directory if it doesn't exist

93 os.execute("mkdir -p " .. model_dir)

94

95 return {

96 embeddings = model_dir .. "/embeddings.json",

97 similarity_matrix = model_dir .. "/similarity_matrix.json",

98 metadata = model_dir .. "/metadata.json"

99 }

104local function cosine_similarity(vec1, vec2)

105 if #vec1 ~= #vec2 then

106 error("Vectors must have same dimension")

107 end

108

109 local dot_product = 0

110 local norm1 = 0

111 local norm2 = 0

112

113 for i = 1, #vec1 do

114 dot_product = dot_product + (vec1[i] * vec2[i])

115 norm1 = norm1 + (vec1[i] * vec1[i])

116 norm2 = norm2 + (vec2[i] * vec2[i])

117 end

118

119 norm1 = math.sqrt(norm1)

120 norm2 = math.sqrt(norm2)

121

122 if norm1 == 0 or norm2 == 0 then

123 return 0

124 end

125

126 return dot_product / (norm1 * norm2)

134local function generate_embedding(text, endpoint, model_name)

135 -- Create a temporary file to avoid shell escaping issues.

136 -- Issue 8-059: route through the project's RAM-backed tmp/shared-memory/

137 -- tier so parallel checkouts of this repository do not collide on a single

138 -- shared /tmp/ filename. (Data, not code — the noexec tier is correct.)

139 os.execute(string.format('"%s/scripts/ensure-tmp-symlink" "%s"', DIR, DIR))

140 local temp_file = DIR .. "/tmp/shared-memory/embedding_input.json"

141 local payload = {

142 model = model_name,

143 -- Apply the active server's task-prefix (e.g. "clustering: " for

144 -- nomic-embed-text v1.5+). No-op for models that don't need one.

145 input = inference_config.format_embedding_prompt(text)

146 }

147

148 local f = io.open(temp_file, "w")

149 if not f then

150 utils.log_error("Failed to create temporary file")

151 return nil, "file_error"

152 end

153 f:write(dkjson.encode(payload))

154 f:close()

155

156 -- 10-049: /v1/embeddings (OpenAI shape) replaces Ollama's /api/embed.

157 -- llama.cpp exposes a single endpoint regardless of which model is

158 -- loaded, so the endpoint path is the same for every model in the

159 -- embedding_models table above (the per-model endpoint_path field

160 -- was removed in the same migration).

161 local cmd = string.format(

162 'curl -s --connect-timeout 10 --max-time 30 "%s/v1/embeddings" -H "Content-Type: application/json" -d @%s',

163 endpoint, temp_file

164 )

165

166 local handle = io.popen(cmd)

167 local result = handle:read("*a")

168 local success, exit_type, exit_code = handle:close()

169

170 -- Clean up temp file

171 os.remove(temp_file)

172

173 -- Check for network/connection errors

174 if not success or exit_code ~= 0 then

175 utils.log_error("Network error: curl failed with exit code " .. (exit_code or "unknown"))

176 return nil, "network_error"

177 end

178

179 -- Check for empty or invalid response

180 if not result or result:match("^%s*$") then

181 utils.log_error("Empty response from API endpoint")

182 return nil, "empty_response"

183 end

184

185 -- Check for curl error messages

186 if result:match("curl:") or result:match("Could not resolve host") or result:match("Connection refused") then

187 utils.log_error("Connection error: " .. result:gsub("\n", " "))

188 return nil, "connection_error"

189 end

190

191 local parsed = dkjson.decode(result)

192 -- 10-049: OpenAI shape — vectors live under data[N].embedding rather

193 -- than directly under .embeddings[N]. We send one input per call here,

194 -- so we read data[1].embedding.

195 if parsed and parsed.data and parsed.data[1] and parsed.data[1].embedding then

196 -- Accept any positive-dimension embedding. The hardcoded "== 768"

197 -- that used to live here would have rejected every output from

198 -- qwen3-embedding (2560-D) or any other non-gemma model. Downstream

199 -- code reads the dimension off the embedding itself rather than

200 -- relying on a fixed value, so there is nothing to gain from

201 -- gating here.

202 local embedding = parsed.data[1].embedding

203 if type(embedding) == "table" and #embedding > 0 then

204 return embedding, "success"

205 else

206 utils.log_error("Invalid embedding response: " .. (type(embedding) == "table" and "empty table" or type(embedding)))

207 return nil, "invalid_dimensions"

208 end

209 else

210 utils.log_error("Failed to parse API response: " .. (result:sub(1, 200) or "nil"))

211 return nil, "parse_error"

212 end

217local function table_length(t)

218 local count = 0

219 for _ in pairs(t) do

220 count = count + 1

221 end

222 return count

229local function generate_random_embedding(poem_id, dimension)

230 dimension = dimension or 768

232 -- Seed with poem_id for reproducibility

233 local seed = type(poem_id) == "number" and poem_id or 12345

234 math.randomseed(seed)

236 local embedding = {}

237 local norm = 0

239 -- Generate random values

240 for i = 1, dimension do

241 embedding[i] = math.random() * 2 - 1 -- Range: -1 to 1

242 norm = norm + embedding[i] * embedding[i]

243 end

245 -- Normalize to unit vector for consistent similarity calculations

246 norm = math.sqrt(norm)

247 if norm > 0 then

248 for i = 1, dimension do

249 embedding[i] = embedding[i] / norm

250 end

251 end

253 return embedding

260local function inherit_embedding(nearest_embedding, own_embedding, dimension)

261 dimension = dimension or 768

263 if not nearest_embedding then

264 return nil -- No embedding to inherit

265 end

267 if not own_embedding then

268 -- Pure inheritance: just copy the nearest embedding

269 local result = {}

270 for i = 1, dimension do

271 result[i] = nearest_embedding[i]

272 end

273 return result

274 end

276 -- Combine embeddings: average of nearest and own

277 -- This gives semantic meaning from context while preserving any content the post has

278 local result = {}

279 local norm = 0

281 for i = 1, dimension do

282 result[i] = (nearest_embedding[i] + own_embedding[i]) / 2

283 norm = norm + result[i] * result[i]

284 end

286 -- Normalize to unit vector for consistent similarity calculations

287 norm = math.sqrt(norm)

288 if norm > 0 then

289 for i = 1, dimension do

290 result[i] = result[i] / norm

291 end

292 end

294 return result

299local network_error_config = {

300 max_consecutive_errors = 5, -- Max consecutive network errors before abort

301 max_total_errors = 20, -- Max total network errors in session

302 initial_retry_delay = 2, -- Initial delay in seconds

303 max_retry_delay = 60, -- Maximum delay in seconds

304 backoff_multiplier = 2 -- Exponential backoff multiplier

305}

309function migrate_legacy_cache(legacy_file, target_model_dir)

310 if utils.file_exists(legacy_file) then

311 utils.log_info("Migrating legacy cache to model-specific storage...")

312

313 local backup_file = legacy_file .. ".legacy_backup"

314 os.rename(legacy_file, backup_file)

315

316 local legacy_data = utils.read_json_file(backup_file)

317 if legacy_data then

318 utils.write_json_file(target_model_dir .. "/embeddings.json", legacy_data)

319 utils.log_info("Legacy cache migrated successfully")

320 end

321 end

326function M.list_available_models()

327 utils.log_info("Available Embedding Models:")

328 for model_name, config in pairs(embedding_models) do

329 utils.log_info(" " .. model_name .. " (" .. config.dimensions .. " dims)")

330 end

331 return embedding_models

336function M.get_model_status(base_output_dir, model_name)

337 -- Default to the configured/overridden model, not a hardcoded literal, so a

338 -- model swap in config.lua (or a --model on the CLI) is reflected here too.

339 model_name = model_name or inference_config.get_selected_model()

340 local storage_paths = get_model_storage_path(base_output_dir, model_name)

341

342 if utils.file_exists(storage_paths.embeddings) then

343 local data = utils.read_json_file(storage_paths.embeddings)

344 if data and data.embeddings then

345 local count = 0

346 for _ in pairs(data.embeddings) do

347 count = count + 1

348 end

349 return {

350 exists = true,

351 count = count,

352 location = storage_paths.embeddings,

353 metadata = data.metadata

354 }

355 end

356 end

357

358 return {

359 exists = false,

360 count = 0,

361 location = storage_paths.embeddings

362 }

367function M.show_all_model_status(base_output_dir)

368 utils.log_info("Available Embedding Models:")

369 for model_name, config in pairs(embedding_models) do

370 local status = M.get_model_status(base_output_dir, model_name)

371 if status.exists then

372 local completion_rate = status.metadata and status.metadata.completion_rate or 0

373 utils.log_info(" " .. model_name .. " (" .. config.dimensions .. " dims) - " ..

374 status.count .. " cached embeddings (" ..

375 string.format("%.1f%%", completion_rate * 100) .. ")")

376 else

377 utils.log_info(" " .. model_name .. " (" .. config.dimensions .. " dims) - No cache found")

378 end

379 end

384function M.generate_all_embeddings(poems_file, base_output_dir, endpoint, incremental, model_name)

385 -- Issue 10-017: Use build_host_url() instead of deprecated OLLAMA_ENDPOINT

386 endpoint = endpoint or inference_config.build_host_url()

387 incremental = incremental ~= false -- Default to true

388 -- Default to the configured/overridden model, not a hardcoded literal (the

389 -- caller, generate-embeddings.sh, always passes one; this guards direct use).

390 model_name = model_name or inference_config.get_selected_model()

391

392 -- Get model-specific configuration

393 local model_config = embedding_models[model_name]

394 if not model_config then

395 utils.log_error("Unknown embedding model: " .. model_name)

396 return false

397 end

398

399 -- Generate model-specific file paths

400 local storage_paths = get_model_storage_path(base_output_dir, model_name)

401 local output_file = storage_paths.embeddings

402

403 utils.log_info("Using embedding model: " .. model_name)

404 utils.log_info("Storage location: " .. output_file)

405 utils.log_info("Expected dimensions: " .. model_config.dimensions)

406

407 -- Handle legacy cache migration

408 local legacy_cache = base_output_dir .. "/embeddings.json"

409 if utils.file_exists(legacy_cache) and output_file ~= legacy_cache then

410 migrate_legacy_cache(legacy_cache, base_output_dir .. "/embeddings/" .. model_name:gsub("[^%w%-_.]", "_"))

411 end

412

413 utils.log_info("Loading poems from: " .. poems_file)

414 local poems_data = utils.read_json_file(poems_file)

415 if not poems_data or not poems_data.poems then

416 utils.log_error("Failed to load poems from " .. poems_file)

417 return false

418 end

419 local poems = poems_data.poems

420

421 -- Load existing embeddings if incremental mode enabled

422 local existing_embeddings = {}

423 -- Pull the dimension from the model registry. If the model is unknown,

424 -- leave dim at nil here; it will be populated from the first embedding

425 -- we actually receive below, so the metadata reflects ground truth.

426 local model_dim = embedding_models[model_name] and embedding_models[model_name].dimensions or nil

427 local embeddings_data = {

428 metadata = {

429 total_poems = #poems,

430 embedding_model = model_name,

431 embedding_dimension = model_dim,

432 generated_at = os.date("%Y-%m-%d %H:%M:%S"),

433 endpoint = endpoint,

434 incremental_update = incremental

435 },

436 embeddings = {}

437 }

438

439 if incremental and utils.file_exists(output_file) then

440 utils.log_info("Incremental mode: Loading existing embeddings...")

441 local existing_data = utils.read_json_file(output_file)

442 if existing_data and existing_data.embeddings then

443 -- Handle both array and object formats for existing embeddings

444 -- Key insight (Issue 8-019): We store by poem_index, not by id, because

445 -- the same id can exist in multiple categories (e.g., fediverse/0002.txt

446 -- and messages/0002.txt both have id=2 but different poem_index values).

447 if type(existing_data.embeddings) == "table" then

448 if existing_data.embeddings[1] then

449 -- Array format (legacy format before poem_index)

450 -- Use poem_index from embedding if available, else use array position

451 for i, emb in ipairs(existing_data.embeddings) do

452 local key = emb.poem_index or i

453 existing_embeddings[key] = emb

454 end

455 else

456 -- Object format (current format) - key-value pairs by poem_index

457 for poem_index, emb in pairs(existing_data.embeddings) do

458 -- Store by poem_index for correct lookup

459 existing_embeddings[tonumber(poem_index)] = emb

460 end

461 end

462 end

464 -- Preserve existing metadata

465 if existing_data.metadata then

466 embeddings_data.metadata.original_generated_at = existing_data.metadata.generated_at

467 embeddings_data.metadata.previous_total = existing_data.metadata.total_poems

468 end

470 utils.log_info("Found " .. table_length(existing_embeddings) .. " existing embeddings")

471 end

472 end

473

474 -- Count poems that need processing

475 local poems_to_process = {}

476 local skipped_count = 0

477 local retry_count = 0

478 local retry_reasons = {}

480 if incremental then

481 -- Incremental mode: Check existing embeddings and only process missing/invalid ones

482 for i, poem in ipairs(poems) do

483 -- Use poem_index if available, fallback to array index for legacy poems.json

484 -- This ensures correct matching even when the same id appears in multiple categories.

485 local lookup_key = poem.poem_index or i

487 -- Only skip if embedding is valid AND dimensions are correct

488 if existing_embeddings[lookup_key] and

489 existing_embeddings[lookup_key].embedding and

490 type(existing_embeddings[lookup_key].embedding) == "table" and

491 #existing_embeddings[lookup_key].embedding == model_config.dimensions then

492 -- Skip: valid embedding found

493 embeddings_data.embeddings[lookup_key] = existing_embeddings[lookup_key]

494 skipped_count = skipped_count + 1

495 else

496 -- Re-process: no embedding, invalid embedding, or error state

497 table.insert(poems_to_process, {index = lookup_key, poem = poem})

499 -- Track retry reasons for reporting

500 if existing_embeddings[lookup_key] then

501 if existing_embeddings[lookup_key].error then

502 retry_count = retry_count + 1

503 local error_type = existing_embeddings[lookup_key].error

504 retry_reasons[error_type] = (retry_reasons[error_type] or 0) + 1

505 elseif existing_embeddings[lookup_key].embedding then

506 -- Invalid embedding dimensions

507 retry_count = retry_count + 1

508 retry_reasons["invalid_dimensions"] = (retry_reasons["invalid_dimensions"] or 0) + 1

509 end

510 end

511 end

512 end

513 end

515 if incremental then

516 utils.log_info("Incremental processing summary:")

517 utils.log_info(" Total poems: " .. #poems)

518 utils.log_info(" Valid existing embeddings: " .. skipped_count)

519

520 -- Enhanced retry reporting

521 if retry_count > 0 then

522 local retry_details = {}

523 for error_type, count in pairs(retry_reasons) do

524 table.insert(retry_details, error_type .. ": " .. count)

525 end

526 utils.log_info(" Error entries to retry: " .. retry_count .. " (" .. table.concat(retry_details, ", ") .. ")")

527 end

528

529 local new_poems = #poems_to_process - retry_count

530 if new_poems > 0 then

531 utils.log_info(" New poems to process: " .. new_poems)

532 end

533

534 utils.log_info(" Processing queue: " .. #poems_to_process .. " poems" ..

535 (retry_count > 0 and (" (" .. new_poems .. " new + " .. retry_count .. " retries)") or ""))

536 utils.log_info(" Processing savings: " .. string.format("%.1f%%", (skipped_count / #poems) * 100))

537

538 if #poems_to_process == 0 then

539 utils.log_info("✅ All embeddings already exist and are valid!")

540 embeddings_data.metadata.completed_embeddings = skipped_count

541 embeddings_data.metadata.completion_rate = 1.0

542 embeddings_data.metadata.processing_mode = "no_update_needed"

543 return utils.write_json_file(output_file, embeddings_data)

544 end

545 else

546 utils.log_info("Full regeneration mode: Processing all " .. #poems .. " poems...")

547 for i, poem in ipairs(poems) do

548 table.insert(poems_to_process, {index = i, poem = poem})

549 end

550 end

551

552 -- Issue 10-050: poems are embedded a WINDOW at a time. All normal text poems

553 -- in a window go out as ONE batched + chunked embedding call (was: one HTTP

554 -- request per poem). Window size is the batch primitive's BATCH_SIZE.

555 local window = fuzzy.BATCH_SIZE

556 if window < 1 then window = 1 end

557 -- Issue 8-021 Fix: Track newly processed poems separately to prevent overcounting.

558 -- The bug occurred when key lookups failed due to poem_index format mismatches,

559 -- causing poems to be added to poems_to_process even though they had valid embeddings

560 -- under different keys. This led to completed = skipped_count + #poems_to_process > #poems.

561 local newly_processed = 0 -- Track only newly processed poems

562 local total_poems = #poems -- Cache for sanity checks

564 -- Sanity check: detect potential key mismatch (Issue 8-021)

565 -- If skipped_count + #poems_to_process > #poems, there's likely a key lookup issue

566 if skipped_count + #poems_to_process > total_poems then

567 utils.log_warn("⚠️ Potential key mismatch detected:")

568 utils.log_warn(" skipped_count (" .. skipped_count .. ") + poems_to_process (" .. #poems_to_process .. ") = " .. (skipped_count + #poems_to_process))

569 utils.log_warn(" This exceeds total poems (" .. total_poems .. ")")

570 utils.log_warn(" Some embeddings may be stored under legacy keys.")

571 utils.log_warn(" Continuing with processing - data will be correct, only counter may be affected.")

572 end

574 -- Network error tracking

575 local consecutive_errors = 0

576 local total_errors = 0

577 local current_delay = network_error_config.initial_retry_delay

578

579 -- Write initial progress state (just counts, no timing)

580 local user = os.getenv("USER") or "ritz" -- fallback to ritz

581 -- Issue 8-059: shared with scripts/generate-embeddings.sh which reads

582 -- this file; both sides now agree on the project-local tmpfs path.

583 local progress_file = DIR .. "/tmp/shared-memory/embedding_progress_" .. user .. ".txt"

584 -- Issue 8-021 Fix: Use safe_completed to cap progress at total_poems

585 local safe_completed = math.min(skipped_count + newly_processed, total_poems)

586 local initial_progress = string.format("%d,%d", safe_completed, total_poems)

587 local pf = io.open(progress_file, "w")

588 if pf then

589 pf:write(initial_progress)

590 pf:close()

591 end

592

718 -- Save the cache roughly every ~100 poems regardless of window size. The old

719 -- `i % 100 == 1` test assumed a step of 10; with a variable window we count

720 -- windows instead so the periodic checkpoint survives a crash mid-run.

721 local windows_since_save = 0

722 local SAVE_EVERY_WINDOWS = math.max(1, math.floor(100 / window))

724 for i = 1, #poems_to_process, window do

725 local batch_end = math.min(i + window - 1, #poems_to_process)

726 utils.log_info(string.format("Processing batch %d-%d of %d new/updated poems...", i, batch_end, #poems_to_process))

728 -- Partition this window: normal text poems get batched together; image-

729 -- only and empty poems are deferred to after the batch resolves.

730 local normal = {}

731 local deferred = {}

732 for j = i, batch_end do

733 local poem_data = poems_to_process[j]

734 local poem = poem_data.poem

735 local poem_index = poem_data.index

736 -- Issue 6-033: enhanced preprocessing for better embedding quality.

737 local poem_text = poem_extractor.extract_pure_poem_content_for_embedding(poem.content)

738 local entry = { poem = poem, poem_index = poem_index, poem_text = poem_text }

739 if poem.is_image_only and poem.nearest_text_poem_index then

740 table.insert(deferred, entry)

741 elseif poem_text == "" then

742 table.insert(deferred, entry)

743 else

744 table.insert(normal, entry)

745 end

746 end

748 -- Embed all normal poems of the window in ONE batched + chunked call.

749 -- A whole-batch transport failure is treated exactly like the old

750 -- per-poem network_error branch: count it, check the thresholds, back

751 -- off, and retry the SAME window.

752 if #normal > 0 then

753 local window_done = false

754 while not window_done do

755 local texts = {}

756 for k = 1, #normal do texts[k] = normal[k].poem_text end

757 utils.log_info(string.format(" Embedding %d text poems (batched, chunked)...", #normal))

758 local vectors, err = fuzzy.embed_texts_with_chunking(texts, model_name, embed_opts)

760 if not vectors then

761 consecutive_errors = consecutive_errors + 1

762 total_errors = total_errors + 1

763 utils.log_warn(string.format("Network error %d/%d for batch %d-%d: %s",

764 consecutive_errors, network_error_config.max_consecutive_errors,

765 i, batch_end, tostring(err)))

767 if consecutive_errors >= network_error_config.max_consecutive_errors then

768 local safe_completed = math.min(skipped_count + newly_processed, total_poems)

769 utils.log_error("❌ NETWORK ERROR THRESHOLD EXCEEDED")

770 utils.log_error(" • Consecutive errors: " .. consecutive_errors .. "/" .. network_error_config.max_consecutive_errors)

771 utils.log_error(" • Poems processed before termination: " .. safe_completed .. "/" .. total_poems)

772 utils.log_error("The embedding cache has been preserved.")

773 embeddings_data.metadata.completed_embeddings = safe_completed

774 embeddings_data.metadata.completion_rate = safe_completed / total_poems

775 embeddings_data.metadata.processing_mode = "terminated_network_error"

776 embeddings_data.metadata.termination_reason = "consecutive_network_errors"

777 embeddings_data.metadata.last_error_count = consecutive_errors

778 utils.write_json_file(output_file, embeddings_data)

779 return false

780 elseif total_errors >= network_error_config.max_total_errors then

781 utils.log_error("❌ TOTAL ERROR LIMIT EXCEEDED")

782 utils.log_error("Too many network errors in this session: " .. total_errors .. "/" .. network_error_config.max_total_errors)

783 return false

784 else

785 utils.log_info("Retrying in " .. current_delay .. " seconds...")

786 os.execute("sleep " .. current_delay)

787 current_delay = math.min(current_delay * network_error_config.backoff_multiplier,

788 network_error_config.max_retry_delay)

789 -- loop again: retry this whole window

790 end

791 else

792 -- Batch produced results: reset error counters (the server is

793 -- alive) and distribute vectors to each poem.

794 consecutive_errors = 0

795 current_delay = network_error_config.initial_retry_delay

796 for k = 1, #normal do

797 local n = normal[k]

798 local embedding = vectors[k]

799 if embedding and type(embedding) == "table" and #embedding == model_config.dimensions then

800 store_success(n.poem, n.poem_index, n.poem_text, embedding)

801 else

802 -- One poem's vector is missing/wrong-dimension. Single-

803 -- retry it once via the same chunk-aware path; if that

804 -- still fails, record a non-critical error so it is not

805 -- retried forever (matches the old `else` branch).

806 local single = fuzzy.embed_texts_with_chunking({ n.poem_text }, model_name, embed_opts)

807 local sv = single and single[1]

808 if sv and type(sv) == "table" and #sv == model_config.dimensions then

809 store_success(n.poem, n.poem_index, n.poem_text, sv)

810 else

811 embeddings_data.embeddings[n.poem_index] = {

812 poem_index = n.poem_index, -- Issue 8-019

813 id = n.poem.id,

814 embedding = nil,

815 error = "embedding_failed",

816 updated_at = os.date("%Y-%m-%d %H:%M:%S")

817 }

818 utils.log_warn("Non-critical error for poem " .. n.poem_index .. ": embedding_failed")

819 end

820 end

821 end

822 write_progress()

823 window_done = true

824 end

825 end

826 end

828 -- Now the deferred poems, with the window's fresh embeddings available.

829 for _, d in ipairs(deferred) do

830 handle_deferred(d.poem, d.poem_index, d.poem_text)

831 end

832 write_progress()

834 -- Periodic cache checkpoint (crash safety on long runs).

835 windows_since_save = windows_since_save + 1

836 if windows_since_save >= SAVE_EVERY_WINDOWS or batch_end == #poems_to_process then

837 windows_since_save = 0

838 local safe_completed = math.min(skipped_count + newly_processed, total_poems)

839 utils.log_info("Saving progress... (" .. newly_processed .. " new + " .. skipped_count .. " existing = " .. safe_completed .. " total)")

840 if not utils.write_json_file(output_file, embeddings_data) then

841 utils.log_error("Failed to save embeddings to " .. output_file)

842 return false

843 end

844 end

845 end

847 -- Issue 8-021 Fix: Use safe calculation for final metadata

848 local safe_completed = math.min(skipped_count + newly_processed, total_poems)

849 embeddings_data.metadata.completed_embeddings = safe_completed

850 embeddings_data.metadata.completion_rate = safe_completed / total_poems

851 embeddings_data.metadata.new_embeddings = newly_processed

852 embeddings_data.metadata.reused_embeddings = skipped_count

853 embeddings_data.metadata.processing_mode = incremental and "incremental" or "full_regeneration"

854 -- Note: timing_data feature was planned but never implemented.

855 -- Removed reference to undefined timing_data variable (Issue 8-018).

857 utils.log_info("Embedding generation complete!")

858 if incremental then

859 utils.log_info("Incremental processing results:")

860 utils.log_info(" New embeddings generated: " .. newly_processed)

861 utils.log_info(" Existing embeddings reused: " .. skipped_count)

862 utils.log_info(" Total embeddings: " .. safe_completed .. " out of " .. total_poems)

863 utils.log_info(" Time savings: " .. string.format("%.1f%%", (skipped_count / total_poems) * 100))

864 else

865 utils.log_info("Full regeneration results:")

866 utils.log_info(" Successfully generated " .. safe_completed .. " out of " .. total_poems .. " embeddings")

867 end

868 utils.log_info("Completion rate: " .. string.format("%.1f%%", (safe_completed / total_poems) * 100))

869

870 return utils.write_json_file(output_file, embeddings_data)

875local function validate_similarity_matrix_currency(similarity_file, embeddings_file, poems_file)

876 if not utils.file_exists(similarity_file) then

877 return {valid = false, reason = "no_matrix_found"}

878 end

879

880 local similarity_data = utils.read_json_file(similarity_file)

881 local embeddings_data = utils.read_json_file(embeddings_file)

882 local poems_data = utils.read_json_file(poems_file)

883

884 if not similarity_data or not similarity_data.metadata then

885 return {valid = false, reason = "no_metadata"}

886 end

887

888 local total_poems = #poems_data.poems

889

890 -- Count current valid embeddings

891 local current_embeddings = 0

892 if embeddings_data and embeddings_data.embeddings then

893 for _, emb in pairs(embeddings_data.embeddings) do

894 if emb.embedding and #emb.embedding > 0 then

895 current_embeddings = current_embeddings + 1

896 end

897 end

898 end

899

900 local matrix_embeddings = similarity_data.metadata.embedding_count or 0

901

902 if current_embeddings ~= matrix_embeddings then

903 return {

904 valid = false,

905 reason = "embedding_count_mismatch",

906 current_count = current_embeddings,

907 matrix_count = matrix_embeddings,

908 difference = current_embeddings - matrix_embeddings

909 }

910 end

911

912 if not similarity_data.metadata.is_complete then

913 return {

914 valid = false,

915 reason = "incomplete_dataset",

916 completeness = similarity_data.metadata.matrix_completeness or 0,

917 missing_embeddings = total_poems - current_embeddings

918 }

919 end

920

921 return {valid = true, metadata = similarity_data.metadata}

949function M.calculate_similarity_matrix(embeddings_file, output_file, top_n, force_regenerate)

950 top_n = top_n or 10

951 force_regenerate = force_regenerate or false

953 -- Need poems file for validation (use configured assets path)

954 local poems_file = utils.asset_path("poems.json")

955

956 -- Validate existing matrix unless forced to regenerate

957 if not force_regenerate then

958 local validation = validate_similarity_matrix_currency(output_file, embeddings_file, poems_file)

959 if validation.valid then

960 utils.log_info("✅ Existing similarity matrix is current and complete")

961 return true

962 else

963 utils.log_warn("⚠️ Similarity matrix validation failed: " .. validation.reason)

964 if validation.reason == "embedding_count_mismatch" then

965 utils.log_info(" Current embeddings: " .. validation.current_count)

966 utils.log_info(" Matrix embeddings: " .. validation.matrix_count)

967 utils.log_info(" Difference: " .. validation.difference)

968 elseif validation.reason == "incomplete_dataset" then

969 utils.log_info(" Completeness: " .. string.format("%.1f%%", validation.completeness * 100))

970 utils.log_info(" Missing embeddings: " .. validation.missing_embeddings)

971 end

972 utils.log_info("🗑️ Removing stale similarity matrix...")

973 os.remove(output_file)

974 end

975 end

976

977 utils.log_info("Loading embeddings from: " .. embeddings_file)

978 local embeddings_data = utils.read_json_file(embeddings_file)

979 if not embeddings_data or not embeddings_data.embeddings then

980 utils.log_error("Failed to load embeddings from " .. embeddings_file)

981 return false

982 end

983

984 local embeddings = embeddings_data.embeddings

985 local valid_embeddings = {}

986

987 -- Filter out invalid embeddings

988 for i, item in ipairs(embeddings) do

989 if item.embedding and #item.embedding > 0 then

990 table.insert(valid_embeddings, {

991 index = i,

992 id = item.id,

993 embedding = item.embedding

994 })

995 end

996 end

997

998 -- Load poems data to get actual total count

999 local poems_data = utils.read_json_file(poems_file)

1000 local total_poems = poems_data and #poems_data.poems or #embeddings

1001

1002 -- Calculate completeness metrics

1003 local embedding_count = #valid_embeddings

1004 local matrix_completeness = embedding_count / total_poems

1005 local is_complete = embedding_count == total_poems

1006

1007 -- Warn about incomplete datasets

1008 if not is_complete then

1009 utils.log_warn("⚠️ WARNING: Incomplete dataset detected")

1010 utils.log_info(" Embeddings: " .. embedding_count .. " / " .. total_poems .. " poems (" .. string.format("%.1f%%", matrix_completeness * 100) .. " complete)")

1011 utils.log_info(" Missing: " .. (total_poems - embedding_count) .. " poems will not appear in recommendations")

1012 utils.log_info("")

1013 utils.log_info(" For complete recommendations, generate embeddings for all poems first")

1014 end

1015

1016 utils.log_info("Calculating similarity matrix for " .. #valid_embeddings .. " valid embeddings...")

1017

1018 local similarity_data = {

1019 metadata = {

1020 generated_at = os.date("%Y-%m-%d %H:%M:%S"),

1021 model_name = embeddings_data.metadata and embeddings_data.metadata.embedding_model or "unknown",

1022 total_poems = total_poems,

1023 embedding_count = embedding_count,

1024 matrix_completeness = matrix_completeness,

1025 is_complete = is_complete,

1026 top_n = top_n,

1027 algorithm = "cosine_similarity"

1028 },

1029 similarities = {}

1030 }

1031

1032 local total_comparisons = #valid_embeddings * (#valid_embeddings - 1) / 2

1033 local completed_comparisons = 0

1034

1035 for i = 1, #valid_embeddings do

1036 local poem_a = valid_embeddings[i]

1037 local similarities_for_poem = {}

1039 -- Issue 8-024: Use carriage return to overwrite line in-place

1040 io.write(string.format("\r[INFO] Processing poem %d/%d (ID: %s) ", i, #valid_embeddings, poem_a.id or "unknown"))

1041 io.flush()

1043 for j = 1, #valid_embeddings do

1044 if i ~= j then

1045 local poem_b = valid_embeddings[j]

1046 local similarity = cosine_similarity(poem_a.embedding, poem_b.embedding)

1047

1048 table.insert(similarities_for_poem, {

1049 id = poem_b.id,

1050 index = poem_b.index,

1051 similarity = similarity

1052 })

1053

1054 if j > i then

1055 completed_comparisons = completed_comparisons + 1

1056 end

1057 end

1058 end

1059

1060 -- Sort by similarity (highest first) and keep only top N

1061 table.sort(similarities_for_poem, function(a, b) return a.similarity > b.similarity end)

1062

1063 local top_similarities = {}

1064 for k = 1, math.min(top_n, #similarities_for_poem) do

1065 table.insert(top_similarities, similarities_for_poem[k])

1066 end

1067

1068 local poem_key = poem_a.id or ("poem_" .. poem_a.index)

1069 similarity_data.similarities[poem_key] = {

1070 poem_index = poem_a.index,

1071 top_similar = top_similarities,

1072 calculated_at = os.date("%Y-%m-%d %H:%M:%S")

1073 }

1074

1075 -- Save progress periodically

1076 if i % 50 == 0 or i == #valid_embeddings then

1077 local progress = (completed_comparisons / total_comparisons) * 100

1078 -- Issue 8-024: Newline before progress to preserve it (processing line uses \r)

1079 io.write("\n")

1080 utils.log_info(string.format("Progress: %.1f%% (%d/%d comparisons)", progress, completed_comparisons, total_comparisons))

1081

1082 if not utils.write_json_file(output_file, similarity_data) then

1083 utils.log_error("Failed to save similarity matrix to " .. output_file)

1084 return false

1085 end

1086 end

1087 end

1088

1089 utils.log_info("Similarity matrix calculation complete!")

1090 utils.log_info("Calculated similarities for " .. #valid_embeddings .. " poems")

1091 utils.log_info("Total comparisons: " .. total_comparisons)

1092

1093 return true

1103function M.calculate_full_similarity_matrix(embeddings_file, output_file, force_regenerate)

1104 force_regenerate = force_regenerate or false

1106 -- Need poems file for validation (use configured assets path)

1107 local poems_file = utils.asset_path("poems.json")

1108

1109 -- Check if full matrix already exists and is current

1110 if not force_regenerate and utils.file_exists(output_file) then

1111 local existing_data = utils.read_json_file(output_file)

1112 if existing_data and existing_data.metadata and existing_data.metadata.is_complete then

1113 utils.log_info("✅ Full similarity matrix already exists and is complete")

1114 return true

1115 end

1116 end

1117

1118 utils.log_info("🔍 Generating FULL similarity matrix (all poem pairs)...")

1119 utils.log_info("⚠️ This will generate ALL 47.1M comparisons (no symmetry optimization) and may take 4-8 hours")

1120

1121 -- Load embeddings

1122 local embeddings_data = utils.read_json_file(embeddings_file)

1123 if not embeddings_data or not embeddings_data.embeddings then

1124 utils.log_error("Failed to load embeddings from " .. embeddings_file)

1125 return false

1126 end

1127

1128 local embeddings = embeddings_data.embeddings

1129 local valid_embeddings = {}

1130

1131 -- Filter out invalid embeddings

1132 for _, embedding in ipairs(embeddings) do

1133 if embedding.embedding and #embedding.embedding > 0 and embedding.id then

1134 table.insert(valid_embeddings, embedding)

1135 end

1136 end

1137

1138 if #valid_embeddings == 0 then

1139 utils.log_error("No valid embeddings found")

1140 return false

1141 end

1142

1143 utils.log_info(string.format("Processing %d poems for full similarity matrix", #valid_embeddings))

1144

1145 local total_comparisons = #valid_embeddings * #valid_embeddings

1146 local completed_comparisons = 0

1147 local start_time = os.time()

1148

1149 -- Initialize full similarity matrix

1150 local similarity_data = {

1151 metadata = {

1152 is_complete = true,

1153 total_poems = #valid_embeddings,

1154 matrix_size = total_comparisons,

1155 algorithm = "cosine_similarity",

1156 model_name = embeddings_data.metadata.embedding_model or "unknown",

1157 generated_at = os.date("%Y-%m-%d %H:%M:%S"),

1158 embedding_count = #valid_embeddings

1159 },

1160 similarities = {}

1161 }

1162

1163 -- Generate COMPLETE similarity matrix (calculate ALL comparisons for maximum accuracy)

1164 for i = 1, #valid_embeddings do

1165 local poem_a = valid_embeddings[i]

1166 local poem_a_id = tostring(poem_a.id)

1167 similarity_data.similarities[poem_a_id] = {}

1169 -- Issue 8-024: Use carriage return to overwrite line in-place

1170 io.write(string.format("\r[INFO] Processing poem %d/%d (ID: %s) ", i, #valid_embeddings, poem_a_id))

1171 io.flush()

1173 for j = 1, #valid_embeddings do

1174 local poem_b = valid_embeddings[j]

1175 local poem_b_id = tostring(poem_b.id)

1176

1177 if i == j then

1178 -- Self-similarity is always 1.0

1179 similarity_data.similarities[poem_a_id][poem_b_id] = 1.0

1180 else

1181 -- Calculate similarity for EVERY comparison (no symmetry optimization)

1182 -- This ensures maximum accuracy by computing each comparison independently

1183 local similarity = cosine_similarity(poem_a.embedding, poem_b.embedding)

1184 -- Round to 4 decimal places for storage efficiency

1185 local rounded_similarity = math.floor(similarity * 10000) / 10000

1186

1187 similarity_data.similarities[poem_a_id][poem_b_id] = rounded_similarity

1188 end

1189

1190 completed_comparisons = completed_comparisons + 1

1191 end

1192

1193 -- Progressive saving every 100 poems to prevent data loss

1194 if i % 100 == 0 or i == #valid_embeddings then

1195 local progress = (completed_comparisons / total_comparisons) * 100

1196 local elapsed_time = os.time() - start_time

1197 local rate = completed_comparisons / elapsed_time

1198 local estimated_remaining = (total_comparisons - completed_comparisons) / rate

1200 -- Issue 8-024: Newline before progress to preserve it (processing line uses \r)

1201 io.write("\n")

1202 utils.log_info(string.format("Progress: %.2f%% (%d/%d comparisons)",

1203 progress, completed_comparisons, total_comparisons))

1204 utils.log_info(string.format("Rate: %.0f comparisons/sec, Est. remaining: %.0f minutes",

1205 rate, estimated_remaining / 60))

1206

1207 if not utils.write_json_file(output_file, similarity_data) then

1208 utils.log_error("Failed to save similarity matrix to " .. output_file)

1209 return false

1210 end

1211 utils.log_info("✅ Progress saved to disk")

1212 end

1213

1214 -- Memory management: force garbage collection periodically

1215 if i % 500 == 0 then

1216 collectgarbage("collect")

1217 end

1218 end

1219

1220 -- Final save with completion timestamp

1221 similarity_data.metadata.completed_at = os.date("%Y-%m-%d %H:%M:%S")

1222 similarity_data.metadata.generation_time_seconds = os.time() - start_time

1223

1224 if not utils.write_json_file(output_file, similarity_data) then

1225 utils.log_error("Failed to save final similarity matrix")

1226 return false

1227 end

1228

1229 utils.log_info("🎉 Full similarity matrix generation complete!")

1230 utils.log_info(string.format("Total comparisons: %d", total_comparisons))

1231 utils.log_info(string.format("Generation time: %.1f minutes", (os.time() - start_time) / 60))

1232 utils.log_info(string.format("Matrix saved to: %s", output_file))

1233

1234 return true

1244function M.calculate_triangular_similarity_matrix(embeddings_file, output_file, force_regenerate)

1245 utils.log_info("🔍 Generating TRIANGULAR similarity matrix (optimized storage)...")

1246

1247 -- Check if output already exists and not forcing regeneration

1248 if not force_regenerate and utils.file_exists(output_file) then

1249 utils.log_info("Triangular similarity matrix already exists. Use force_regenerate=true to recreate.")

1250 return true

1251 end

1252

1253 local embeddings_data = utils.read_json_file(embeddings_file)

1254 if not embeddings_data or not embeddings_data.embeddings then

1255 utils.log_error("Failed to load embeddings file: " .. embeddings_file)

1256 return false

1257 end

1258

1259 local embeddings = embeddings_data.embeddings

1260 local poems = {}

1261

1262 -- Filter out invalid embeddings (same as full matrix function)

1263 for _, embedding in ipairs(embeddings) do

1264 if embedding.embedding and #embedding.embedding > 0 and embedding.id then

1265 table.insert(poems, embedding)

1266 end

1267 end

1268

1269 if #poems == 0 then

1270 utils.log_error("No valid embeddings found")

1271 return false

1272 end

1273

1274 utils.log_info("Processing " .. #poems .. " poems for triangular similarity matrix")

1275

1276 -- Calculate storage requirements

1277 local total_unique_pairs = (#poems * (#poems - 1)) / 2

1278 utils.log_info(string.format("⚠️ This will generate %d unique comparisons (50%% reduction from full matrix)", total_unique_pairs))

1279 utils.log_info("⚠️ Expected storage: ~50% reduction from full matrix size")

1280

1281 local similarity_data = {

1282 metadata = {

1283 matrix_size = total_unique_pairs,

1284 total_poems = #poems,

1285 model_name = embeddings_data.model_name,

1286 algorithm = "cosine_similarity",

1287 embedding_count = #poems,

1288 generated_at = os.date("%Y-%m-%d %H:%M:%S"),

1289 is_complete = true,

1290 storage_format = "triangular_upper"

1291 },

1292 similarities = {}

1293 }

1294

1295 local start_time = os.time()

1296 local completed = 0

1297

1298 -- Generate upper triangular matrix only (i < j)

1299 for i = 1, #poems do

1300 local poem_a = poems[i]

1301 similarity_data.similarities[tostring(poem_a.id)] = {}

1302

1303 -- Only calculate similarities for j > i (upper triangle)

1304 for j = i + 1, #poems do

1305 local poem_b = poems[j]

1306

1307 local similarity = cosine_similarity(poem_a.embedding, poem_b.embedding)

1308 similarity_data.similarities[tostring(poem_a.id)][tostring(poem_b.id)] =

1309 math.floor(similarity * 10000) / 10000 -- 4 decimal precision

1310

1311 completed = completed + 1

1312

1313 -- Progress reporting every 10000 comparisons

1314 if completed % 10000 == 0 then

1315 local progress_percent = (completed / total_unique_pairs) * 100

1316 local elapsed = os.time() - start_time

1317 local rate = completed / elapsed

1318 local remaining_time = (total_unique_pairs - completed) / rate / 60

1319

1320 utils.log_info(string.format("Progress: %.2f%% (%d/%d comparisons)",

1321 progress_percent, completed, total_unique_pairs))

1322 utils.log_info(string.format("Rate: %.0f comparisons/sec, Est. remaining: %.1f minutes",

1323 rate, remaining_time))

1324 end

1325 end

1326

1327 -- Progressive saving every 100 poems

1328 if i % 100 == 0 then

1329 utils.write_json_file(output_file, similarity_data)

1330 utils.log_info(string.format("✅ Progress saved to disk (poem %d/%d)", i, #poems))

1331 end

1332

1333 -- Garbage collection every 500 poems

1334 if i % 500 == 0 then

1335 collectgarbage()

1336 utils.log_info(string.format("🗑️ Memory cleanup completed (poem %d/%d)", i, #poems))

1337 end

1338 end

1339

1340 -- Final save

1341 if not utils.write_json_file(output_file, similarity_data) then

1342 utils.log_error("Failed to save triangular similarity matrix")

1343 return false

1344 end

1345

1346 utils.log_info("✅ TRIANGULAR similarity matrix generation completed!")

1347 utils.log_info(string.format("Total unique comparisons: %d", total_unique_pairs))

1348 utils.log_info(string.format("Generation time: %.1f minutes", (os.time() - start_time) / 60))

1349 utils.log_info(string.format("Matrix saved to: %s", output_file))

1350 utils.log_info("📊 Storage optimized: ~50% reduction from full matrix")

1351

1352 return true

1357function M.get_similarity_triangular(matrix, id1, id2)

1358 -- Handle diagonal (self-similarity)

1359 if id1 == id2 then return 1.0 end

1360

1361 -- Ensure consistent ordering for triangle lookup (min_id -> max_id)

1362 local min_id = math.min(tonumber(id1), tonumber(id2))

1363 local max_id = math.max(tonumber(id1), tonumber(id2))

1364

1365 -- Look up in upper triangle

1366 if matrix.similarities[tostring(min_id)] and

1367 matrix.similarities[tostring(min_id)][tostring(max_id)] then

1368 return matrix.similarities[tostring(min_id)][tostring(max_id)]

1369 end

1370

1371 -- Fallback (should not happen with complete matrix)

1372 utils.log_warning(string.format("Similarity not found for poems %s and %s", id1, id2))

1373 return 0.0

1378function M.get_all_similarities_for_poem_triangular(matrix, poem_id, poem_ids)

1379 local similarities = {}

1380

1381 for _, other_id in ipairs(poem_ids) do

1382 if other_id ~= poem_id then

1383 local score = M.get_similarity_triangular(matrix, poem_id, other_id)

1384 table.insert(similarities, {

1385 target_id = other_id,

1386 score = score

1387 })

1388 end

1389 end

1390

1391 -- Sort by similarity score (descending)

1392 table.sort(similarities, function(a, b)

1393 return a.score > b.score

1394 end)

1395

1396 return similarities

1401function M.generate_similarity_report(similarity_file, poems_file, output_file)

1402 utils.log_info("Generating similarity analysis report...")

1403

1404 local similarity_data = utils.read_json_file(similarity_file)

1405 local poems_data = utils.read_json_file(poems_file)

1406

1407 if not similarity_data or not poems_data then

1408 utils.log_error("Failed to load required data files")

1409 return false

1410 end

1411

1412 local report = {

1413 metadata = {

1414 generated_at = os.date("%Y-%m-%d %H:%M:%S"),

1415 total_poems = #poems_data,

1416 poems_with_similarities = 0,

1417 average_similarity = 0,

1418 max_similarity = 0,

1419 min_similarity = 1

1420 },

1421 statistics = {},

1422 sample_similarities = {}

1423 }

1424

1425 local total_similarity = 0

1426 local similarity_count = 0

1427

1428 for poem_id, data in pairs(similarity_data.similarities) do

1429 report.metadata.poems_with_similarities = report.metadata.poems_with_similarities + 1

1430

1431 if data.top_similar and #data.top_similar > 0 then

1432 local max_sim = data.top_similar[1].similarity

1433 local min_sim = data.top_similar[#data.top_similar].similarity

1434

1435 report.metadata.max_similarity = math.max(report.metadata.max_similarity, max_sim)

1436 report.metadata.min_similarity = math.min(report.metadata.min_similarity, min_sim)

1437

1438 for _, sim in ipairs(data.top_similar) do

1439 total_similarity = total_similarity + sim.similarity

1440 similarity_count = similarity_count + 1

1441 end

1442

1443 -- Add sample for high-similarity pairs

1444 if max_sim > 0.8 then

1445 table.insert(report.sample_similarities, {

1446 poem_a_id = poem_id,

1447 poem_b_id = data.top_similar[1].id,

1448 similarity = max_sim

1449 })

1450 end

1451 end

1452 end

1453

1454 if similarity_count > 0 then

1455 report.metadata.average_similarity = total_similarity / similarity_count

1456 end

1457

1458 utils.log_info("Similarity analysis complete!")

1459 utils.log_info("Poems with similarities: " .. report.metadata.poems_with_similarities)

1460 utils.log_info("Average similarity: " .. string.format("%.3f", report.metadata.average_similarity))

1461 utils.log_info("Similarity range: " .. string.format("%.3f - %.3f", report.metadata.min_similarity, report.metadata.max_similarity))

1462

1463 return utils.write_json_file(output_file, report)

1468function M.generate_all_model_similarity_matrices(base_output_dir, min_completeness, use_full_matrix)

1469 min_completeness = min_completeness or 0.8 -- 80% minimum completeness

1470 use_full_matrix = use_full_matrix or false -- Default to sparse matrices

1471

1472 utils.log_info("🔄 Generating similarity matrices for all eligible models...")

1473 utils.log_info("⚙️ Minimum completeness required: " .. (min_completeness * 100) .. "%")

1474 utils.log_info("📊 Matrix type: " .. (use_full_matrix and "FULL (all comparisons)" or "SPARSE (top-N)"))

1475

1476 local models = M.list_available_models()

1477 local results = {}

1478 local eligible_count = 0

1479 local total_poems = 6860 -- Known total poem count

1480

1481 -- First pass: check eligibility

1482 for model_name, config in pairs(models) do

1483 local status = M.get_model_status(base_output_dir, model_name)

1484

1485 if status.exists then

1486 local completeness = status.count / total_poems

1487

1488 if completeness >= min_completeness then

1489 eligible_count = eligible_count + 1

1490 utils.log_info("✅ " .. model_name .. " (" .. string.format("%.1f%% complete, %d poems)", completeness * 100, status.count) .. ")")

1491 else

1492 utils.log_warn("⚠️ Skipping " .. model_name ..

1493 " (only " .. string.format("%.1f%% complete, %d poems)", completeness * 100, status.count) .. ")")

1494 end

1495 else

1496 utils.log_info("❌ No embeddings found for " .. model_name)

1497 end

1498 end

1499

1500 if eligible_count == 0 then

1501 utils.log_warn("No models meet the minimum completeness requirement")

1502 return {}

1503 end

1504

1505 utils.log_info("📈 Processing " .. eligible_count .. " eligible models")

1506

1507 local current_model = 0

1508

1509 -- Second pass: generate matrices

1510 for model_name, config in pairs(models) do

1511 local status = M.get_model_status(base_output_dir, model_name)

1512

1513 if status.exists then

1514 local completeness = status.count / total_poems

1515

1516 if completeness >= min_completeness then

1517 current_model = current_model + 1

1518

1519 utils.log_info(string.format("🔄 [%d/%d] Processing %s", current_model, eligible_count, model_name))

1520

1521 local storage_paths = get_model_storage_path(base_output_dir, model_name)

1522 local matrix_file = use_full_matrix and

1523 storage_paths.similarity_matrix:gsub("%.json$", "_full.json") or

1524 storage_paths.similarity_matrix

1525

1526 local start_time = os.time()

1527 local success

1528

1529 if use_full_matrix then

1530 success = M.calculate_full_similarity_matrix(

1531 storage_paths.embeddings,

1532 matrix_file,

1533 false -- Don't force regenerate unless needed

1534 )

1535 else

1536 success = M.calculate_similarity_matrix(

1537 storage_paths.embeddings,

1538 matrix_file

1539 )

1540 end

1541

1542 local generation_time = os.time() - start_time

1543

1544 results[model_name] = {

1545 success = success,

1546 completeness = completeness,

1547 embedding_count = status.count,

1548 matrix_file = matrix_file,

1549 generation_time = generation_time,

1550 matrix_type = use_full_matrix and "full" or "sparse"

1551 }

1552

1553 if success then

1554 utils.log_info(string.format("✅ Matrix generation complete for %s (took %d seconds)", model_name, generation_time))

1555 else

1556 utils.log_error("❌ Matrix generation failed for " .. model_name)

1557 end

1558 else

1559 results[model_name] = {

1560 success = false,

1561 reason = "insufficient_completeness",

1562 completeness = completeness,

1563 embedding_count = status.count,

1564 required_completeness = min_completeness

1565 }

1566 end

1567 else

1568 results[model_name] = {

1569 success = false,

1570 reason = "no_embeddings",

1571 completeness = 0,

1572 embedding_count = 0

1573 }

1574 end

1575 end

1576

1577 -- Summary report

1578 local successful_models = 0

1579 local skipped_models = 0

1580 local failed_models = 0

1581

1582 for model_name, result in pairs(results) do

1583 if result.success then

1584 successful_models = successful_models + 1

1585 elseif result.reason then

1586 skipped_models = skipped_models + 1

1587 else

1588 failed_models = failed_models + 1

1589 end

1590 end

1591

1592 utils.log_info("📊 Generation Summary:")

1593 utils.log_info(" ✅ Successful: " .. successful_models .. " models")

1594 utils.log_info(" ⚠️ Skipped: " .. skipped_models .. " models")

1595 utils.log_info(" ❌ Failed: " .. failed_models .. " models")

1596

1597 return results

1602function M.compare_model_similarities(poem_id, base_output_dir, models, use_full_matrix)

1603 use_full_matrix = use_full_matrix or false

1604 models = models or {}

1605

1606 -- If no models specified, use all available models

1607 if #models == 0 then

1608 local available_models = M.list_available_models()

1609 for model_name, _ in pairs(available_models) do

1610 table.insert(models, model_name)

1611 end

1612 end

1613

1614 utils.log_info("🔍 Comparing similarities for poem " .. poem_id .. " across models")

1615

1616 local comparisons = {}

1617

1618 for _, model_name in ipairs(models) do

1619 local storage_paths = get_model_storage_path(base_output_dir, model_name)

1620 local matrix_file = use_full_matrix and

1621 storage_paths.similarity_matrix:gsub("%.json$", "_full.json") or

1622 storage_paths.similarity_matrix

1623

1624 if utils.file_exists(matrix_file) then

1625 -- For now, generate basic similarity data - this would integrate with recommendation system

1626 comparisons[model_name] = {

1627 matrix_available = true,

1628 matrix_type = use_full_matrix and "full" or "sparse",

1629 matrix_file = matrix_file

1630 }

1631 utils.log_info("✅ " .. model_name .. " - Matrix available")

1632 else

1633 comparisons[model_name] = {

1634 matrix_available = false,

1635 reason = "matrix_not_found"

1636 }

1637 utils.log_info("❌ " .. model_name .. " - Matrix not found")

1638 end

1639 end

1640

1641 return comparisons

1646function M.get_multi_model_status(base_output_dir)

1647 utils.log_info("📊 Per-Model Similarity Matrix Status:")

1648

1649 local models = M.list_available_models()

1650 local total_poems = 6860

1651 local status_summary = {}

1652

1653 for model_name, config in pairs(models) do

1654 local status = M.get_model_status(base_output_dir, model_name)

1655 local storage_paths = get_model_storage_path(base_output_dir, model_name)

1656

1657 local sparse_matrix_exists = utils.file_exists(storage_paths.similarity_matrix)

1658 local full_matrix_file = storage_paths.similarity_matrix:gsub("%.json$", "_full.json")

1659 local full_matrix_exists = utils.file_exists(full_matrix_file)

1660

1661 local completeness = status.exists and (status.count / total_poems) or 0

1662

1663 utils.log_info(" " .. model_name .. " (" .. config.dimensions .. " dims)")

1664

1665 if status.exists then

1666 utils.log_info(string.format(" ✅ Embeddings: %d/%d (%.1f%%)",

1667 status.count, total_poems, completeness * 100))

1668 else

1669 utils.log_info(" ❌ Embeddings: 0/" .. total_poems .. " (0%)")

1670 end

1671

1672 if sparse_matrix_exists then

1673 utils.log_info(" ✅ Sparse Matrix: Generated")

1674 else

1675 utils.log_info(" ❌ Sparse Matrix: Not generated")

1676 end

1677

1678 if full_matrix_exists then

1679 utils.log_info(" ✅ Full Matrix: Generated")

1680 else

1681 utils.log_info(" ❌ Full Matrix: Not generated")

1682 end

1683

1684 if completeness < 0.8 then

1685 local needed = math.ceil((0.8 * total_poems) - status.count)

1686 utils.log_info(" 🔄 Recommendation: Complete " .. needed .. " more embeddings")

1687 end

1688

1689 status_summary[model_name] = {

1690 dimensions = config.dimensions,

1691 embedding_count = status.count,

1692 completeness = completeness,

1693 sparse_matrix_exists = sparse_matrix_exists,

1694 full_matrix_exists = full_matrix_exists,

1695 eligible_for_generation = completeness >= 0.8

1696 }

1697 end

1698

1699 return status_summary

1704function M.main(interactive_mode)

1705 if interactive_mode then

1706 utils.log_info("=== Similarity Engine Interactive Mode ===")

1707 print("1. Generate embeddings for all poems")

1708 print("2. Calculate similarity matrix (sparse, top-N)")

1709 print("3. Calculate FULL similarity matrix (all pairs)")

1710 print("4. Generate similarity analysis report")

1711 print("5. Run complete pipeline")

1712 print("6. Generate matrices for ALL eligible models")

1713 print("7. Show multi-model status")

1714 print("8. Compare model similarities")

1715 io.write("Select option (1-8): ")

1716 local choice = io.read()

1717

1718 if choice == "1" then

1719 local poems_file = utils.asset_path("poems.json")

1720 local base_output_dir = utils.get_assets_root()

1721 io.write("Use incremental processing? (Y/n): ")

1722 local incremental_choice = io.read()

1723 local incremental = not (incremental_choice:lower() == "n" or incremental_choice:lower() == "no")

1724 io.write("Embedding model (default: EmbeddingGemma:latest): ")

1725 local model_input = io.read()

1726 local model_name = model_input ~= "" and model_input or "embeddinggemma:latest"

1727 M.generate_all_embeddings(poems_file, base_output_dir, nil, incremental, model_name)

1728 elseif choice == "2" then

1729 io.write("Embedding model (default: EmbeddingGemma:latest): ")

1730 local model_input = io.read()

1731 local model_name = model_input ~= "" and model_input or "embeddinggemma:latest"

1732 local base_output_dir = utils.get_assets_root()

1733 local storage_paths = get_model_storage_path(base_output_dir, model_name)

1734 local embeddings_file = storage_paths.embeddings

1735 local output_file = storage_paths.similarity_matrix

1736 M.calculate_similarity_matrix(embeddings_file, output_file)

1737 elseif choice == "3" then

1738 io.write("Embedding model (default: EmbeddingGemma:latest): ")

1739 local model_input = io.read()

1740 local model_name = model_input ~= "" and model_input or "embeddinggemma:latest"

1741 local base_output_dir = utils.get_assets_root()

1742 local storage_paths = get_model_storage_path(base_output_dir, model_name)

1743 local embeddings_file = storage_paths.embeddings

1744 local output_file = storage_paths.similarity_matrix:gsub("%.json$", "_full.json")

1746 utils.log_info("⚠️ FULL matrix generation will take 2-4 hours and create ~100MB file")

1747 io.write("Continue? (y/N): ")

1748 local confirm = io.read()

1749 if confirm:lower() == "y" or confirm:lower() == "yes" then

1750 M.calculate_full_similarity_matrix(embeddings_file, output_file, false)

1751 else

1752 utils.log_info("Full matrix generation cancelled")

1753 end

1754 elseif choice == "4" then

1755 local similarity_file = utils.asset_path("similarity-matrix.json")

1756 local poems_file = utils.asset_path("poems.json")

1757 local output_file = utils.asset_path("similarity-report.json")

1758 M.generate_similarity_report(similarity_file, poems_file, output_file)

1759 elseif choice == "5" then

1760 utils.log_info("Running complete similarity engine pipeline...")

1761 local poems_file = utils.asset_path("poems.json")

1762 local base_output_dir = utils.get_assets_root()

1763 local similarity_file = utils.asset_path("similarity-matrix.json")

1764 local report_file = utils.asset_path("similarity-report.json")

1766 if M.generate_all_embeddings(poems_file, base_output_dir) then

1767 local storage_paths = get_model_storage_path(base_output_dir, "embeddinggemma:latest")

1768 local embeddings_file = storage_paths.embeddings

1769 if M.calculate_similarity_matrix(embeddings_file, similarity_file) then

1770 M.generate_similarity_report(similarity_file, poems_file, report_file)

1771 utils.log_info("✅ Complete pipeline executed successfully!")

1772 else

1773 utils.log_error("Pipeline failed at similarity matrix calculation")

1774 end

1775 else

1776 utils.log_error("Pipeline failed at embedding generation")

1777 end

1778 elseif choice == "6" then

1779 local base_output_dir = utils.get_assets_root()

1780 io.write("Matrix type - (s)parse or (f)ull? (default: sparse): ")

1781 local matrix_type = io.read()

1782 local use_full_matrix = matrix_type:lower():sub(1,1) == "f"

1784 io.write("Minimum completeness percentage (default: 80): ")

1785 local completeness_input = io.read()

1786 local min_completeness = tonumber(completeness_input) or 80

1787 min_completeness = min_completeness / 100 -- Convert percentage to decimal

1789 local results = M.generate_all_model_similarity_matrices(base_output_dir, min_completeness, use_full_matrix)

1790 utils.log_info("Multi-model generation complete. Results available in similarity engine.")

1791 elseif choice == "7" then

1792 local base_output_dir = utils.get_assets_root()

1793 M.get_multi_model_status(base_output_dir)

1794 elseif choice == "8" then

1795 io.write("Poem ID to compare: ")

1796 local poem_id = tonumber(io.read())

1797 local base_output_dir = utils.get_assets_root()

1798 io.write("Use (s)parse or (f)ull matrices? (default: sparse): ")

1799 local matrix_type = io.read()

1800 local use_full_matrix = matrix_type:lower():sub(1,1) == "f"

1802 local results = M.compare_model_similarities(poem_id, base_output_dir, {}, use_full_matrix)

1803 utils.log_info("Model comparison complete.")

1804 else

1805 print("Invalid choice")

1806 end

1807 else

1808 -- Default: run similarity analysis on existing data

1809 utils.log_info("Running similarity engine analysis...")

1810 -- Issue 8-032: Fixed filename inconsistency (was similarity-matrix.json with hyphen)

1811 local similarity_file = utils.asset_path("similarity_matrix.json")

1812 local poems_file = utils.asset_path("poems.json")

1813 local report_file = utils.asset_path("similarity-report.json")

1815 if utils.file_exists(similarity_file) then

1816 M.generate_similarity_report(similarity_file, poems_file, report_file)

1817 else

1818 utils.log_info("No similarity matrix found. Use interactive mode (-I) to generate embeddings and similarities.")

1819 end

1820 end

1825function M.flush_embeddings_cache(output_file, flush_type, backup)

1826 flush_type = flush_type or "all" -- "all", "errors", "model_specific"

1827 backup = backup ~= false -- Default to true

1828

1829 if not utils.file_exists(output_file) then

1830 utils.log_info("No cache file found at: " .. output_file)

1831 return true

1832 end

1833

1834 -- Get file info for reporting

1835 local file_size = os.execute("du -h '" .. output_file .. "' 2>/dev/null") and

1836 io.popen("du -h '" .. output_file .. "' | cut -f1"):read("*l") or "unknown"

1837

1838 utils.log_info("Cache flush operation: " .. flush_type)

1839 utils.log_info("Target file: " .. output_file)

1840 utils.log_info("File size: " .. file_size)

1841

1842 if backup then

1843 local backup_file = output_file .. ".backup." .. os.date("%Y%m%d_%H%M%S")

1844

1845 -- Use Lua file operations for better cross-platform compatibility

1846 local source_file = io.open(output_file, "rb")

1847 if not source_file then

1848 utils.log_error("Failed to open source file for backup")

1849 return false

1850 end

1851

1852 local content = source_file:read("*a")

1853 source_file:close()

1854

1855 local backup_dest = io.open(backup_file, "wb")

1856 if not backup_dest then

1857 utils.log_error("Failed to create backup file: " .. backup_file)

1858 return false

1859 end

1860

1861 backup_dest:write(content)

1862 backup_dest:close()

1863

1864 utils.log_info("Backup created: " .. backup_file)

1865 end

1866

1867 if flush_type == "all" then

1868 -- Complete cache flush

1869 local remove_result = os.remove(output_file)

1870 if remove_result then

1871 utils.log_info("✅ Complete embedding cache flushed")

1872 return true

1873 else

1874 utils.log_error("Failed to remove cache file")

1875 return false

1876 end

1877

1878 elseif flush_type == "errors" then

1879 -- Flush only error entries, keep valid embeddings

1880 local existing_data = utils.read_json_file(output_file)

1881 if not existing_data or not existing_data.embeddings then

1882 utils.log_warn("No embeddings data found in cache file")

1883 return true

1884 end

1885

1886 local clean_embeddings = {}

1887 local removed_count = 0

1888 local kept_count = 0

1889

1890 -- "Valid" means "matches the cache file's declared dimension." That

1891 -- declared dimension comes from the metadata block of this same

1892 -- file, written when the cache was first created — so this check

1893 -- is model-agnostic now (was hardcoded to 768 for embeddinggemma).

1894 local expected_dim = existing_data.metadata and existing_data.metadata.embedding_dimension

1895 for i, emb in pairs(existing_data.embeddings) do

1896 local dim_ok = emb.embedding and type(emb.embedding) == "table"

1897 and #emb.embedding > 0

1898 and (not expected_dim or #emb.embedding == expected_dim)

1899 if dim_ok then

1900 -- Keep valid embeddings

1901 clean_embeddings[i] = emb

1902 kept_count = kept_count + 1

1903 else

1904 -- Remove error entries

1905 removed_count = removed_count + 1

1906 end

1907 end

1908

1909 existing_data.embeddings = clean_embeddings

1910

1911 -- Update metadata

1912 if existing_data.metadata then

1913 existing_data.metadata.completed_embeddings = kept_count

1914 existing_data.metadata.last_flush_operation = {

1915 type = "errors_only",

1916 timestamp = os.date("%Y-%m-%d %H:%M:%S"),

1917 removed_entries = removed_count,

1918 kept_entries = kept_count

1919 }

1920 end

1921

1922 local write_success = utils.write_json_file(output_file, existing_data)

1923 if write_success then

1924 utils.log_info("✅ Error entries flushed: " .. removed_count .. " entries removed, " .. kept_count .. " kept")

1925 return true

1926 else

1927 utils.log_error("Failed to write cleaned cache file")

1928 return false

1929 end

1930

1931 else

1932 utils.log_error("Unknown flush type: " .. flush_type)

1933 return false

1934 end