src/validation-engine.lua

19function ValidationEngine:new(config)

20 config = config or {}

21 local obj = {

22 config = config,

23 calculator = nil, -- Will be set from modular calculator

24 tolerance = config.tolerance or 0.001,

25 sample_size = config.sample_size or nil, -- nil = validate all

26 progress_callback = config.progress_callback or nil,

27 validation_results = {

28 total_comparisons = 0,

29 accurate_scores = 0,

30 inaccurate_scores = 0,

31 missing_embeddings = 0,

32 errors = {},

33 discrepancies = {},

34 start_time = nil,

35 end_time = nil

36 }

37 }

38

39 setmetatable(obj, ValidationEngine)

40 return obj

45function ValidationEngine:set_calculator(calculator)

46 self.calculator = calculator

51function ValidationEngine:validate_similarity_matrix(similarity_file, embeddings_file)

52 if not self.calculator then

53 error("Similarity calculator must be set before validation")

54 end

55

56 self.validation_results.start_time = os.time()

57 print(string.format("Starting validation: %s vs %s", similarity_file, embeddings_file))

58

59 -- Load data

60 local similarity_data = utils.read_json_file(similarity_file)

61 local embeddings_data = utils.read_json_file(embeddings_file)

62

63 if not similarity_data or not embeddings_data then

64 error("Failed to load validation data files")

65 end

66

67 -- Create validation sample

68 local validation_pairs = self:create_validation_sample(similarity_data, embeddings_data)

69

70 print(string.format("Validating %d similarity pairs...", #validation_pairs))

71

72 -- Validate each pair

73 local progress_interval = math.max(1, math.floor(#validation_pairs / 20))

74

75 for i, pair in ipairs(validation_pairs) do

76 if i % progress_interval == 0 then

77 print(string.format("Progress: %d/%d (%.1f%%)", i, #validation_pairs, (i/#validation_pairs)*100))

78 if self.progress_callback then

79 self.progress_callback(i, #validation_pairs)

80 end

81 end

82

83 self:validate_similarity_pair(pair, embeddings_data)

84 end

85

86 self.validation_results.end_time = os.time()

87

88 return self:generate_validation_report()

93function ValidationEngine:validate_similarity_pair(pair, embeddings_data)

94 local poem_a_id, poem_b_id, stored_score = pair.poem_a, pair.poem_b, pair.stored_score

95

96 self.validation_results.total_comparisons = self.validation_results.total_comparisons + 1

97

98 -- Get embeddings

99 local embedding_a = embeddings_data[tostring(poem_a_id)]

100 local embedding_b = embeddings_data[tostring(poem_b_id)]

101

102 if not embedding_a or not embedding_b then

103 self.validation_results.missing_embeddings = self.validation_results.missing_embeddings + 1

104 table.insert(self.validation_results.errors, {

105 type = "missing_embedding",

106 poem_a = poem_a_id,

107 poem_b = poem_b_id,

108 missing = not embedding_a and "poem_a" or "poem_b"

109 })

110 return

111 end

112

113 -- Calculate actual similarity

114 local success, calculated_score = pcall(function()

115 return self.calculator:calculate(embedding_a, embedding_b)

116 end)

117

118 if not success then

119 table.insert(self.validation_results.errors, {

120 type = "calculation_error",

121 poem_a = poem_a_id,

122 poem_b = poem_b_id,

123 error = calculated_score -- This will contain the error message

124 })

125 return

126 end

127

128 -- Compare scores

129 local difference = math.abs(calculated_score - stored_score)

130

131 if difference <= self.tolerance then

132 self.validation_results.accurate_scores = self.validation_results.accurate_scores + 1

133 else

134 self.validation_results.inaccurate_scores = self.validation_results.inaccurate_scores + 1

135 table.insert(self.validation_results.discrepancies, {

136 poem_a = poem_a_id,

137 poem_b = poem_b_id,

138 stored_score = stored_score,

139 calculated_score = calculated_score,

140 difference = difference,

141 relative_error = math.abs(stored_score) > 0 and (difference / math.abs(stored_score)) or 0

142 })

143 end

148function ValidationEngine:create_validation_sample(similarity_data, embeddings_data)

149 local all_pairs = {}

150

151 -- Extract all similarity pairs from data

152 for poem_a_id, similarities in pairs(similarity_data) do

153 if type(similarities) == "table" then

154 for poem_b_id, score in pairs(similarities) do

155 table.insert(all_pairs, {

156 poem_a = tonumber(poem_a_id),

157 poem_b = tonumber(poem_b_id),

158 stored_score = tonumber(score)

159 })

160 end

161 end

162 end

163

164 print(string.format("Found %d total similarity pairs", #all_pairs))

165

166 -- Apply sampling if requested

167 if self.sample_size and self.sample_size < #all_pairs then

168 print(string.format("Sampling %d pairs for validation", self.sample_size))

169

170 -- Random sampling

171 local sampled_pairs = {}

172 local used_indices = {}

173

174 math.randomseed(os.time())

175

176 while #sampled_pairs < self.sample_size do

177 local random_index = math.random(1, #all_pairs)

178 if not used_indices[random_index] then

179 table.insert(sampled_pairs, all_pairs[random_index])

180 used_indices[random_index] = true

181 end

182 end

183

184 return sampled_pairs

185 else

186 return all_pairs

187 end

192function ValidationEngine:generate_validation_report()

193 local results = self.validation_results

194 local duration = results.end_time - results.start_time

195

196 -- Calculate statistics

197 local accuracy_rate = results.total_comparisons > 0 and

198 (results.accurate_scores / results.total_comparisons) or 0

199

200 local error_rate = results.total_comparisons > 0 and

201 (results.inaccurate_scores / results.total_comparisons) or 0

202

203 local report = {

204 algorithm = self.calculator and self.calculator.algorithm or "unknown",

205 timestamp = os.date("%Y-%m-%d %H:%M:%S", results.start_time),

206 duration_seconds = duration,

207 statistics = {

208 total_comparisons = results.total_comparisons,

209 accurate_scores = results.accurate_scores,

210 inaccurate_scores = results.inaccurate_scores,

211 missing_embeddings = results.missing_embeddings,

212 accuracy_rate = accuracy_rate,

213 error_rate = error_rate,

214 tolerance = self.tolerance

215 },

216 performance = {

217 comparisons_per_second = duration > 0 and (results.total_comparisons / duration) or 0,

218 avg_comparison_time_ms = duration > 0 and (duration * 1000 / results.total_comparisons) or 0

219 },

220 discrepancies = {

221 count = #results.discrepancies,

222 samples = self:get_worst_discrepancies(10),

223 max_difference = self:get_max_discrepancy(),

224 avg_difference = self:get_average_discrepancy()

225 },

226 errors = {

227 count = #results.errors,

228 by_type = self:group_errors_by_type(),

229 samples = results.errors

230 },

231 recommendations = self:generate_recommendations()

232 }

233

234 return report

239function ValidationEngine:get_worst_discrepancies(limit)

240 local sorted_discrepancies = {}

241 for _, disc in ipairs(self.validation_results.discrepancies) do

242 table.insert(sorted_discrepancies, disc)

243 end

244

245 -- Sort by difference (highest first)

246 table.sort(sorted_discrepancies, function(a, b)

247 return a.difference > b.difference

248 end)

249

250 local result = {}

251 for i = 1, math.min(limit, #sorted_discrepancies) do

252 table.insert(result, sorted_discrepancies[i])

253 end

254

255 return result

260function ValidationEngine:get_max_discrepancy()

261 local max_diff = 0

262 for _, disc in ipairs(self.validation_results.discrepancies) do

263 if disc.difference > max_diff then

264 max_diff = disc.difference

265 end

266 end

267 return max_diff > 0 and max_diff or nil

272function ValidationEngine:get_average_discrepancy()

273 if #self.validation_results.discrepancies == 0 then

274 return nil

275 end

276

277 local total_diff = 0

278 for _, disc in ipairs(self.validation_results.discrepancies) do

279 total_diff = total_diff + disc.difference

280 end

281

282 return total_diff / #self.validation_results.discrepancies

287function ValidationEngine:group_errors_by_type()

288 local grouped = {}

289 for _, error in ipairs(self.validation_results.errors) do

290 if not grouped[error.type] then

291 grouped[error.type] = 0

292 end

293 grouped[error.type] = grouped[error.type] + 1

294 end

295 return grouped

300function ValidationEngine:generate_recommendations()

301 local results = self.validation_results

302 local recommendations = {}

303

304 local accuracy_rate = results.total_comparisons > 0 and

305 (results.accurate_scores / results.total_comparisons) or 0

306

307 if accuracy_rate < 0.95 then

308 table.insert(recommendations, "Low accuracy rate detected. Consider investigating calculation differences or updating stored similarity data.")

309 end

310

311 if results.missing_embeddings > 0 then

312 table.insert(recommendations, string.format("%d missing embeddings found. Update embedding data or clean similarity matrix.", results.missing_embeddings))

313 end

314

315 if #results.errors > 0 then

316 table.insert(recommendations, string.format("%d calculation errors occurred. Check embedding data quality and calculator implementation.", #results.errors))

317 end

318

319 local max_diff = self:get_max_discrepancy()

320 if max_diff and max_diff > 0.1 then

321 table.insert(recommendations, string.format("Maximum discrepancy of %.4f detected. Consider tightening tolerance or investigating calculation method.", max_diff))

322 end

323

324 if accuracy_rate > 0.99 then

325 table.insert(recommendations, "Excellent accuracy rate. Stored similarity data appears reliable.")

326 end

327

328 return recommendations

333local function run_comprehensive_validation(similarity_files, embeddings_files, algorithms, output_dir)

334 local comprehensive_results = {

335 timestamp = os.date("%Y-%m-%d %H:%M:%S"),

336 algorithms_tested = {},

337 overall_statistics = {

338 total_files = #similarity_files,

339 total_algorithms = #algorithms,

340 successful_validations = 0,

341 failed_validations = 0

342 },

343 file_results = {}

344 }

345

346 for _, algorithm in ipairs(algorithms) do

347 print(string.format("Testing algorithm: %s", algorithm))

348

349 local calculator = SimilarityCalculator:new(algorithm, {cache_enabled = true})

350 local engine = ValidationEngine:new({

351 tolerance = 0.001,

352 sample_size = 1000 -- Sample for large datasets

353 })

354 engine:set_calculator(calculator)

355

356 local algorithm_results = {

357 algorithm = algorithm,

358 files_validated = 0,

359 total_accuracy = 0,

360 validations = {}

361 }

362

363 for i, similarity_file in ipairs(similarity_files) do

364 local embeddings_file = embeddings_files[i]

365

366 print(string.format("Validating file %d/%d with %s", i, #similarity_files, algorithm))

367

368 local success, validation_result = pcall(function()

369 return engine:validate_similarity_matrix(similarity_file, embeddings_file)

370 end)

371

372 if success then

373 table.insert(algorithm_results.validations, validation_result)

374 algorithm_results.files_validated = algorithm_results.files_validated + 1

375 algorithm_results.total_accuracy = algorithm_results.total_accuracy + validation_result.statistics.accuracy_rate

376 comprehensive_results.overall_statistics.successful_validations = comprehensive_results.overall_statistics.successful_validations + 1

377 else

378 print(string.format("Validation failed: %s", validation_result))

379 comprehensive_results.overall_statistics.failed_validations = comprehensive_results.overall_statistics.failed_validations + 1

380 end

381 end

382

383 -- Calculate average accuracy for algorithm

384 algorithm_results.average_accuracy = algorithm_results.files_validated > 0 and

385 (algorithm_results.total_accuracy / algorithm_results.files_validated) or 0

386

387 table.insert(comprehensive_results.algorithms_tested, algorithm_results)

388 end

389

390 -- Generate comprehensive report

391 local report_file = output_dir .. "/validation_comprehensive_report.json"

392 utils.write_json_file(report_file, comprehensive_results)

393

394 print(string.format("Comprehensive validation complete. Report saved: %s", report_file))

395

396 return comprehensive_results

401local function create_validation_engine(config)

402 return ValidationEngine:new(config)

407local function validate_single_file(similarity_file, embeddings_file, algorithm, config)

408 local calculator = SimilarityCalculator:new(algorithm or "cosine", {})

409 local engine = ValidationEngine:new(config or {})

410 engine:set_calculator(calculator)

411

412 return engine:validate_similarity_matrix(similarity_file, embeddings_file)