src/centroid-generator.lua

13local function setup_dir_path(provided_dir)

14 if provided_dir then

15 return provided_dir

16 end

17 return "/mnt/mtwo/programming/ai-stuff/neocities-modernization"

53local _selected_model = inference_config.get_selected_model()

54local CONFIG = {

55 model_name = _selected_model,

56 model_storage_name = _selected_model:gsub("[^%w%-_.]", "_"),

57 embedding_dimensions = nil, -- resolved at runtime from embeddings.json metadata

58 max_content_length = 20000,

59 min_content_length = 10

60}

72local function generate_embedding_with_chunking(text, endpoint, depth)

73 -- Exact token-aware chunking (Issue 10-050): size each chunk by the model's

74 -- real tokenizer (via /tokenize) so it fits the context with no truncation,

75 -- exactly as the poem path does. Raises if /tokenize is unreachable — no

76 -- silent fallback, since the embed call would fail next anyway.

77 local count_fn = fuzzy.make_token_counter(endpoint)

78 local max_tokens = fuzzy.embedding_chunk_budget(endpoint, inference_config.format_embedding_prompt)

79 local chunks = text_chunking.chunk_text_by_tokens(text, count_fn, max_tokens)

80 if #chunks == 0 then

81 utils.log_error("Centroid text produced no chunks (empty after preprocessing)")

82 return nil, "empty_text"

83 end

85 -- All chunks of this centroid embed in one request. endpoint + prompt

86 -- formatter are passed so the centroid shares the poems' embedding space.

87 local vectors, err = fuzzy.get_embeddings_batch(

88 chunks, CONFIG.model_name, endpoint, inference_config.format_embedding_prompt)

89 if not vectors then

90 utils.log_error("Centroid embedding batch failed: " .. tostring(err))

91 return nil, err or "batch_failed"

92 end

94 local all_embeddings = {}

95 for i = 1, #chunks do

96 local v = vectors[i]

97 if type(v) == "table" and #v > 0 then

98 table.insert(all_embeddings, v)

99 -- Learn the model's dimension from the first real vector (matches

100 -- the old behavior; CONFIG.embedding_dimensions is a logging hint).

101 if not CONFIG.embedding_dimensions then

102 CONFIG.embedding_dimensions = #v

103 end

104 else

105 utils.log_warn(string.format(" Missing vector for centroid chunk %d/%d", i, #chunks))

106 end

107 end

109 if #all_embeddings == 0 then

110 return nil, "no_embeddings_generated"

111 end

112 return all_embeddings, "success"

118local function calculate_ultra_centroid(chunk_embeddings)

119 if not chunk_embeddings or #chunk_embeddings == 0 then

120 return nil

121 end

123 -- If only one embedding, just normalize and return it

124 if #chunk_embeddings == 1 then

125 local embedding = chunk_embeddings[1]

126 local magnitude = 0

127 for i = 1, #embedding do

128 magnitude = magnitude + embedding[i] * embedding[i]

129 end

130 magnitude = math.sqrt(magnitude)

132 local normalized = {}

133 for i = 1, #embedding do

134 normalized[i] = embedding[i] / magnitude

135 end

136 return normalized

137 end

139 local dim = #chunk_embeddings[1]

140 local centroid = {}

142 -- Initialize with zeros

143 for i = 1, dim do

144 centroid[i] = 0

145 end

147 -- Sum all chunk embeddings

148 -- Note: Division by count before normalization is unnecessary because

149 -- normalization rescales to unit length regardless of input magnitude.

150 -- See Issue 9-003 for mathematical proof of cosine scale-invariance.

151 for _, embedding in ipairs(chunk_embeddings) do

152 for i = 1, dim do

153 centroid[i] = centroid[i] + embedding[i]

154 end

155 end

157 -- Normalize to unit length (makes any prior scaling irrelevant)

158 local magnitude = 0

159 for i = 1, dim do

160 magnitude = magnitude + centroid[i] * centroid[i]

161 end

162 magnitude = math.sqrt(magnitude)

164 if magnitude > 0 then

165 for i = 1, dim do

166 centroid[i] = centroid[i] / magnitude

167 end

168 end

170 return centroid

176local function load_source_files(file_paths)

177 local contents = {}

179 for _, filepath in ipairs(file_paths or {}) do

180 local content, err = utils.read_file(filepath)

181 if content then

182 table.insert(contents, content)

183 utils.log_info(" Loaded source file: " .. filepath .. " (" .. #content .. " chars)")

184 else

185 utils.log_warn(" Could not load source file: " .. filepath .. " - " .. (err or "unknown error"))

186 end

187 end

189 return table.concat(contents, "\n\n")

195local function build_centroid_text(centroid_def)

196 local parts = {}

198 -- Add source file contents

199 local file_content = load_source_files(centroid_def.source_files)

200 if #file_content > 0 then

201 table.insert(parts, file_content)

202 end

204 -- Add keywords as natural phrases

205 if centroid_def.keywords and #centroid_def.keywords > 0 then

206 local keywords_text = table.concat(centroid_def.keywords, "\n")

207 table.insert(parts, keywords_text)

208 end

210 return table.concat(parts, "\n\n")

216local function generate_centroid_embedding(centroid_def, endpoint)

217 utils.log_info("Processing centroid: " .. centroid_def.name)

219 -- Build combined text

220 local combined_text = build_centroid_text(centroid_def)

222 if #combined_text < CONFIG.min_content_length then

223 utils.log_error(" Combined content too short (" .. #combined_text .. " chars) - need at least " .. CONFIG.min_content_length)

224 return nil, "content_too_short"

225 end

227 utils.log_info(" Combined content: " .. #combined_text .. " chars")

229 -- Generate embeddings (with chunking if needed)

230 local chunk_embeddings, status = generate_embedding_with_chunking(combined_text, endpoint)

232 if not chunk_embeddings then

233 utils.log_error(" Failed to generate embeddings: " .. (status or "unknown"))

234 return nil, status

235 end

237 utils.log_info(" Generated " .. #chunk_embeddings .. " chunk embedding(s)")

239 -- Calculate ultra-centroid

240 local centroid_vector = calculate_ultra_centroid(chunk_embeddings)

242 if not centroid_vector then

243 utils.log_error(" Failed to calculate ultra-centroid")

244 return nil, "centroid_calculation_failed"

245 end

247 utils.log_info(" Ultra-centroid calculated successfully")

249 return {

250 name = centroid_def.name,

251 description = centroid_def.description,

252 output_slug = centroid_def.output_slug,

253 embedding = centroid_vector,

254 chunk_count = #chunk_embeddings,

255 content_length = #combined_text,

256 generated_at = utils.get_timestamp()

257 }, "success"

263function M.generate_all_centroids(options)

264 options = options or {}

266 -- Check inference server availability

267 local endpoint = inference_config.build_host_url()

268 utils.log_info("Using inference endpoint: " .. endpoint)

270 -- Verify endpoint is reachable. /v1/models is llama.cpp's OpenAI-

271 -- compatible liveness probe (was /api/tags under Ollama).

272 local test_cmd = "curl -s --max-time 5 " .. endpoint .. "/v1/models > /dev/null 2>&1"

273 local test_result = os.execute(test_cmd)

274 if test_result ~= 0 and test_result ~= true then

275 utils.log_error("Cannot reach the inference endpoint: " .. endpoint)

276 utils.log_error("Please ensure the inference server is running and accessible.")

277 return nil, "inference_unavailable"

278 end

280 -- Issue 10-003: Load centroids from unified config instead of assets/centroids.json

281 local centroids_list = unified_config.centroids

282 if not centroids_list or #centroids_list == 0 then

283 utils.log_error("No centroids defined in config.lua")

284 return nil, "config_parse_error"

285 end

287 utils.log_info("Found " .. #centroids_list .. " centroid definition(s)")

289 -- Process each centroid

290 local results = {

291 centroids = {},

292 metadata = {

293 model = CONFIG.model_name,

294 dimensions = CONFIG.embedding_dimensions,

295 generated_at = utils.get_timestamp(),

296 source_config = "config.lua"

297 }

298 }

300 local success_count = 0

301 local error_count = 0

303 for i, centroid_def in ipairs(centroids_list) do

304 utils.log_info(string.format("\n[%d/%d] Processing: %s", i, #centroids_list, centroid_def.name))

306 local result, status = generate_centroid_embedding(centroid_def, endpoint)

308 if result then

309 results.centroids[centroid_def.output_slug] = result

310 success_count = success_count + 1

311 else

312 error_count = error_count + 1

313 utils.log_error(" Skipping centroid due to error: " .. (status or "unknown"))

314 end

315 end

317 utils.log_info(string.format("\nGeneration complete: %d succeeded, %d failed", success_count, error_count))

319 -- Determine output path.

320 -- Issue 10-054: route through embeddings_dir() so centroids follow the

321 -- RAM/disk switch like every other movable cache. Equivalent to the old

322 -- assets_root/embeddings/<model_storage_name> while the switch is off (same

323 -- selected model, same sanitization).

324 local output_dir = utils.embeddings_dir()

325 os.execute("mkdir -p " .. output_dir)

327 local output_file = output_dir .. "/centroids.json"

329 -- Save results

330 local json_output = dkjson.encode(results, {indent = true})

331 local write_success, write_err = utils.write_file(output_file, json_output)

333 if not write_success then

334 utils.log_error("Failed to save centroids: " .. (write_err or "unknown"))

335 return nil, "write_error"

336 end

338 utils.log_info("Centroids saved to: " .. output_file)

340 return results, "success"

345if arg and arg[0] and arg[0]:match("centroid%-generator%.lua$") then

346 utils.log_info("=== Centroid Generator ===")

347 utils.log_info("Generating custom mood/theme centroids for exploration pages")

348 utils.log_info("")

350 local results, status = M.generate_all_centroids()

352 if results then

353 utils.log_info("\n=== Summary ===")

354 local count = 0

355 for slug, data in pairs(results.centroids) do

356 count = count + 1

357 utils.log_info(string.format(" %s: %s (%d chunks)", slug, data.name, data.chunk_count))

358 end

359 utils.log_info(string.format("Total: %d centroids generated", count))

360 else

361 utils.log_error("Centroid generation failed: " .. (status or "unknown"))

362 os.exit(1)

363 end