src/triangular-similarity-matrix.lua

38local function cosine_similarity(vec1, vec2)

39 if not vec1 or not vec2 or #vec1 == 0 or #vec2 == 0 then

40 return 0.0

41 end

43 if #vec1 ~= #vec2 then

44 utils.log_error(string.format("Vector dimension mismatch: %d vs %d", #vec1, #vec2))

45 return 0.0

46 end

48 local dot_product = 0.0

49 local norm1 = 0.0

50 local norm2 = 0.0

52 for i = 1, #vec1 do

53 dot_product = dot_product + (vec1[i] * vec2[i])

54 norm1 = norm1 + (vec1[i] * vec1[i])

55 norm2 = norm2 + (vec2[i] * vec2[i])

56 end

58 local magnitude = math.sqrt(norm1) * math.sqrt(norm2)

59 if magnitude == 0 then

60 return 0.0

61 end

63 return dot_product / magnitude

73function M.generate_triangular_matrix(embeddings_file, output_file, force_regenerate, progress_callback)

74 force_regenerate = force_regenerate or false

76 -- Check if matrix already exists

77 if not force_regenerate and utils.file_exists(output_file) then

78 local existing_data = utils.read_json_file(output_file)

79 if existing_data and existing_data.metadata and existing_data.metadata.is_complete then

80 utils.log_info("✅ Triangular similarity matrix already exists and is complete")

81 return true, {exists = true}

82 end

83 end

85 utils.log_info("🔺 Generating triangular similarity matrix...")

86 utils.log_info(" Algorithm: Upper triangle only (i < j)")

87 utils.log_info(" Storage optimization: ~50% size reduction via symmetry")

89 -- Load embeddings

90 local embeddings_data = utils.read_json_file(embeddings_file)

91 if not embeddings_data or not embeddings_data.embeddings then

92 utils.log_error("Failed to load embeddings from " .. embeddings_file)

93 return false, {error = "embeddings_load_failed"}

94 end

96 local embeddings = embeddings_data.embeddings

97 local valid_embeddings = {}

99 -- Filter and index embeddings by ID

100 for _, embedding in ipairs(embeddings) do

101 if embedding.embedding and #embedding.embedding > 0 and embedding.id then

102 valid_embeddings[tonumber(embedding.id)] = embedding

103 end

104 end

106 if next(valid_embeddings) == nil then

107 utils.log_error("No valid embeddings found")

108 return false, {error = "no_valid_embeddings"}

109 end

111 -- Get sorted poem IDs for consistent ordering

112 local poem_ids = {}

113 for id, _ in pairs(valid_embeddings) do

114 table.insert(poem_ids, id)

115 end

116 table.sort(poem_ids)

118 local num_poems = #poem_ids

119 local total_comparisons = (num_poems * (num_poems - 1)) / 2 -- Upper triangle only

120 local completed_comparisons = 0

121 local start_time = os.time()

123 utils.log_info(string.format("Processing %d poems for triangular matrix", num_poems))

124 utils.log_info(string.format("Total comparisons: %.1fM (vs %.1fM for full matrix)",

125 total_comparisons / 1000000, (num_poems * num_poems) / 1000000))

127 -- Initialize triangular matrix

128 local triangular_matrix = {

129 metadata = {

130 is_complete = true,

131 total_poems = num_poems,

132 matrix_type = "upper_triangular",

133 total_comparisons = total_comparisons,

134 algorithm = "cosine_similarity",

135 model_name = embeddings_data.metadata.embedding_model or "unknown",

136 generated_at = os.date("%Y-%m-%d %H:%M:%S"),

137 storage_optimization = "50% reduction via symmetry"

138 },

139 similarities = {}

140 }

142 -- Generate ONLY upper triangle (i < j)

143 for i = 1, num_poems do

144 local poem_i_id = poem_ids[i]

145 local poem_i = valid_embeddings[poem_i_id]

146 triangular_matrix.similarities[tostring(poem_i_id)] = {}

148 -- Progress indicator (carriage return overwrites)

149 io.write(string.format("\r[INFO] Processing poem %d/%d (ID: %d) ",

150 i, num_poems, poem_i_id))

151 io.flush()

153 -- Only calculate for j > i (upper triangle)

154 for j = i + 1, num_poems do

155 local poem_j_id = poem_ids[j]

156 local poem_j = valid_embeddings[poem_j_id]

158 -- Calculate similarity

159 local similarity = cosine_similarity(poem_i.embedding, poem_j.embedding)

160 -- Round to 4 decimal places for storage efficiency

161 local rounded_similarity = math.floor(similarity * 10000) / 10000

163 triangular_matrix.similarities[tostring(poem_i_id)][tostring(poem_j_id)] = rounded_similarity

164 completed_comparisons = completed_comparisons + 1

165 end

167 -- Progressive saving every 100 poems to prevent data loss

168 if i % 100 == 0 then

169 local elapsed = os.time() - start_time

170 local rate = completed_comparisons / elapsed

171 local remaining = (total_comparisons - completed_comparisons) / rate

172 local progress_pct = (completed_comparisons / total_comparisons) * 100

174 print() -- Newline after the carriage-return line

175 utils.log_info(string.format("Progress: %.2f%% (%.1fM/%.1fM comparisons)",

176 progress_pct, completed_comparisons / 1000000, total_comparisons / 1000000))

177 utils.log_info(string.format("Rate: %d comparisons/sec, Est. remaining: %d minutes",

178 math.floor(rate), math.floor(remaining / 60)))

180 -- Write intermediate checkpoint

181 utils.write_json_file(output_file, triangular_matrix)

182 utils.log_info("✅ Progress saved to disk")

184 -- Call progress callback if provided

185 if progress_callback then

186 progress_callback(completed_comparisons, total_comparisons)

187 end

188 end

189 end

191 -- Final save

192 print() -- Newline after last carriage-return

193 local success = utils.write_json_file(output_file, triangular_matrix)

195 if success then

196 local elapsed = os.time() - start_time

197 utils.log_info("✅ Triangular similarity matrix generated successfully!")

198 utils.log_info(string.format("Total comparisons: %.1fM", completed_comparisons / 1000000))

199 utils.log_info(string.format("Time elapsed: %d minutes", math.floor(elapsed / 60)))

200 utils.log_info(string.format("Output: %s", output_file))

202 return true, {

203 comparisons = completed_comparisons,

204 poems = num_poems,

205 elapsed_seconds = elapsed,

206 output_file = output_file

207 }

208 else

209 utils.log_error("Failed to write triangular matrix file")

210 return false, {error = "write_failed"}

211 end

221function M.lookup_similarity(matrix, id1, id2)

222 id1 = tostring(id1)

223 id2 = tostring(id2)

225 -- Handle self-similarity

226 if id1 == id2 then

227 return 1.0

228 end

230 -- Ensure consistent ordering for triangle lookup

231 local min_id = id1

232 local max_id = id2

233 if tonumber(id1) > tonumber(id2) then

234 min_id = id2

235 max_id = id1

236 end

238 -- Look up in upper triangle

239 if matrix.similarities and matrix.similarities[min_id] and matrix.similarities[min_id][max_id] then

240 return matrix.similarities[min_id][max_id]

241 end

243 -- Fallback (should not happen with complete matrix)

244 return 0.0

254function M.get_all_similarities_for_poem(matrix, poem_id, all_poem_ids)

255 local similarities = {}

257 for _, other_id in ipairs(all_poem_ids) do

258 if other_id ~= poem_id then

259 local score = M.lookup_similarity(matrix, poem_id, other_id)

260 table.insert(similarities, {id = other_id, similarity = score})

261 end

262 end

264 -- Sort by similarity (descending)

265 table.sort(similarities, function(a, b)

266 return a.similarity > b.similarity

267 end)

269 return similarities