src/test-validation-engine.lua

17function test_validation_engine_basic()

18 print("🧪 Testing Basic Validation Engine Functionality")

19 print("===============================================")

20

21 -- Create a mock validation engine

22 local engine = ValidationEngine:new({tolerance = 0.01})

23 local calculator = SimilarityCalculator:new("cosine")

24 engine:set_calculator(calculator)

25

26 -- Test engine creation

27 if engine.tolerance == 0.01 and engine.calculator then

28 print("✅ Engine creation and calculator assignment: PASSED")

29 else

30 print("❌ Engine creation and calculator assignment: FAILED")

31 return false

32 end

33

34 -- Test validation results structure

35 local results = engine.validation_results

36 if results.total_comparisons == 0 and

37 type(results.errors) == "table" and

38 type(results.discrepancies) == "table" then

39 print("✅ Validation results structure: PASSED")

40 else

41 print("❌ Validation results structure: FAILED")

42 return false

43 end

44

45 return true

50function test_validation_with_mock_data()

51 print("\n🔬 Testing Validation with Mock Data")

52 print("===================================")

53

54 -- Create mock similarity and embedding data

55 local mock_similarity_data = {

56 ["1"] = {

57 ["2"] = 0.85,

58 ["3"] = 0.42

59 },

60 ["2"] = {

61 ["3"] = 0.67

62 }

63 }

64

65 local mock_embeddings_data = {

66 ["1"] = {0.8, 0.6, 0.0},

67 ["2"] = {0.9, 0.4, 0.1},

68 ["3"] = {0.3, 0.7, 0.9}

69 }

70

71 -- Write mock data to temporary files

72 local temp_similarity_file = DIR .. "/test_similarity.json"

73 local temp_embeddings_file = DIR .. "/test_embeddings.json"

74

75 utils.write_json_file(temp_similarity_file, mock_similarity_data)

76 utils.write_json_file(temp_embeddings_file, mock_embeddings_data)

77

78 -- Test validation

79 local engine = ValidationEngine:new({tolerance = 0.1})

80 local calculator = SimilarityCalculator:new("cosine")

81 engine:set_calculator(calculator)

82

83 local success, report = pcall(function()

84 return engine:validate_similarity_matrix(temp_similarity_file, temp_embeddings_file)

85 end)

86

87 -- Clean up temporary files

88 os.remove(temp_similarity_file)

89 os.remove(temp_embeddings_file)

90

91 if success and report then

92 print(string.format("✅ Mock validation completed successfully"))

93 print(string.format(" - Total comparisons: %d", report.statistics.total_comparisons))

94 print(string.format(" - Accuracy rate: %.2f%%", report.statistics.accuracy_rate * 100))

95 print(string.format(" - Duration: %d seconds", report.duration_seconds))

96 return true

97 else

98 print(string.format("❌ Mock validation failed: %s", report or "unknown error"))

99 return false

100 end

105function test_validation_with_real_data()

106 print("\n📊 Testing Validation with Real Project Data")

107 print("============================================")

108

109 -- Check if real data files exist

110 local similarity_file = DIR .. "/assets/embeddings/embeddinggemma_latest/similarity_matrix.json"

111 local embeddings_file = DIR .. "/assets/embeddings/embeddinggemma_latest/embeddings.json"

112

113 local similarity_exists = utils.file_exists(similarity_file)

114 local embeddings_exists = utils.file_exists(embeddings_file)

115

116 if not similarity_exists or not embeddings_exists then

117 print("⚠️ Real data files not found, skipping real data test")

118 print(string.format(" - Similarity file exists: %s", similarity_exists and "yes" or "no"))

119 print(string.format(" - Embeddings file exists: %s", embeddings_exists and "yes" or "no"))

120 return true -- Not a failure, just no data available

121 end

122

123 print("📁 Real data files found, running validation test...")

124

125 -- Test with sample of real data

126 local engine = ValidationEngine:new({

127 tolerance = 0.001,

128 sample_size = 100 -- Small sample for testing

129 })

130 local calculator = SimilarityCalculator:new("cosine")

131 engine:set_calculator(calculator)

132

133 local success, report = pcall(function()

134 return engine:validate_similarity_matrix(similarity_file, embeddings_file)

135 end)

136

137 if success and report then

138 print(string.format("✅ Real data validation completed successfully"))

139 print(string.format(" - Total comparisons: %d", report.statistics.total_comparisons))

140 print(string.format(" - Accuracy rate: %.2f%%", report.statistics.accuracy_rate * 100))

141 print(string.format(" - Missing embeddings: %d", report.statistics.missing_embeddings))

142 print(string.format(" - Calculation errors: %d", report.errors.count))

143 print(string.format(" - Discrepancies: %d", report.discrepancies.count))

144 print(string.format(" - Performance: %.1f comparisons/sec", report.performance.comparisons_per_second))

145

146 if #report.recommendations > 0 then

147 print(" - Recommendations:")

148 for i, rec in ipairs(report.recommendations) do

149 print(string.format(" %d. %s", i, rec))

150 end

151 end

152

153 return true

154 else

155 print(string.format("❌ Real data validation failed: %s", report or "unknown error"))

156 return false

157 end

162function test_multiple_algorithms()

163 print("\n🧮 Testing Multiple Similarity Algorithms")

164 print("==========================================")

165

166 local algorithms_to_test = {"cosine", "euclidean", "angular", "manhattan"}

167

168 -- Create simple test data

169 local test_data = {

170 similarity = {["1"] = {["2"] = 0.85}},

171 embeddings = {["1"] = {1, 0, 0}, ["2"] = {0.8, 0.6, 0}}

172 }

173

174 local temp_similarity_file = DIR .. "/test_algorithms_similarity.json"

175 local temp_embeddings_file = DIR .. "/test_algorithms_embeddings.json"

176

177 utils.write_json_file(temp_similarity_file, test_data.similarity)

178 utils.write_json_file(temp_embeddings_file, test_data.embeddings)

179

180 local results = {}

181

182 for _, algorithm in ipairs(algorithms_to_test) do

183 print(string.format("Testing algorithm: %s", algorithm))

184

185 local engine = ValidationEngine:new({tolerance = 0.5}) -- Relaxed tolerance for different algorithms

186 local calculator = SimilarityCalculator:new(algorithm)

187 engine:set_calculator(calculator)

188

189 local success, report = pcall(function()

190 return engine:validate_similarity_matrix(temp_similarity_file, temp_embeddings_file)

191 end)

192

193 if success then

194 results[algorithm] = {

195 success = true,

196 accuracy = report.statistics.accuracy_rate,

197 comparisons = report.statistics.total_comparisons

198 }

199 print(string.format(" ✅ %s: %.1f%% accuracy", algorithm, report.statistics.accuracy_rate * 100))

200 else

201 results[algorithm] = {

202 success = false,

203 error = report

204 }

205 print(string.format(" ❌ %s: failed", algorithm))

206 end

207 end

208

209 -- Clean up

210 os.remove(temp_similarity_file)

211 os.remove(temp_embeddings_file)

212

213 local successful_algorithms = 0

214 for _, result in pairs(results) do

215 if result.success then

216 successful_algorithms = successful_algorithms + 1

217 end

218 end

219

220 print(string.format("\nMulti-algorithm test results: %d/%d algorithms successful",

221 successful_algorithms, #algorithms_to_test))

222

223 return successful_algorithms == #algorithms_to_test

228function test_error_handling()

229 print("\n🚨 Testing Error Handling")

230 print("=========================")

231

232 local tests_passed = 0

233 local total_tests = 3

234

235 -- Test 1: Missing calculator

236 print("Test 1: Missing calculator")

237 local engine = ValidationEngine:new()

238 local success, error = pcall(function()

239 engine:validate_similarity_matrix("dummy.json", "dummy.json")

240 end)

241

242 if not success and error:match("calculator must be set") then

243 print(" ✅ Correctly detected missing calculator")

244 tests_passed = tests_passed + 1

245 else

246 print(" ❌ Failed to detect missing calculator")

247 end

248

249 -- Test 2: Invalid data files

250 print("Test 2: Invalid data files")

251 engine:set_calculator(SimilarityCalculator:new("cosine"))

252 success, error = pcall(function()

253 engine:validate_similarity_matrix("nonexistent.json", "nonexistent.json")

254 end)

255

256 if not success then

257 print(" ✅ Correctly handled missing data files")

258 tests_passed = tests_passed + 1

259 else

260 print(" ❌ Failed to handle missing data files")

261 end

262

263 -- Test 3: Malformed data handling

264 print("Test 3: Malformed data handling")

265

266 -- Create files with malformed data

267 local bad_similarity_file = DIR .. "/test_bad_similarity.json"

268 local bad_embeddings_file = DIR .. "/test_bad_embeddings.json"

269

270 utils.write_json_file(bad_similarity_file, {["1"] = {["2"] = "not_a_number"}})

271 utils.write_json_file(bad_embeddings_file, {["1"] = {1, 2, 3}, ["2"] = "not_an_array"})

272

273 success, error = pcall(function()

274 return engine:validate_similarity_matrix(bad_similarity_file, bad_embeddings_file)

275 end)

276

277 -- Clean up

278 os.remove(bad_similarity_file)

279 os.remove(bad_embeddings_file)

280

281 if success then

282 print(" ✅ Gracefully handled malformed data")

283 tests_passed = tests_passed + 1

284 else

285 print(" ❌ Failed to handle malformed data gracefully")

286 end

287

288 print(string.format("Error handling tests: %d/%d passed", tests_passed, total_tests))

289 return tests_passed == total_tests

294function main()

295 print("🔍 Validation Engine Test Suite")

296 print("==============================\n")

297

298 local tests = {

299 {"Basic Functionality", test_validation_engine_basic},

300 {"Mock Data Validation", test_validation_with_mock_data},

301 {"Real Data Validation", test_validation_with_real_data},

302 {"Multiple Algorithms", test_multiple_algorithms},

303 {"Error Handling", test_error_handling}

304 }

305

306 local passed_tests = 0

307 local total_tests = #tests

308

309 for i, test in ipairs(tests) do

310 local test_name, test_func = test[1], test[2]

311

312 local success, result = pcall(test_func)

313

314 if success and result then

315 passed_tests = passed_tests + 1

316 print(string.format("\n✅ %s: PASSED", test_name))

317 else

318 print(string.format("\n❌ %s: FAILED - %s", test_name, result or "unknown error"))

319 end

320 end

321

322 print(string.format("\n\n📊 Final Results: %d/%d tests passed", passed_tests, total_tests))

323

324 if passed_tests == total_tests then

325 print("🎉 All validation engine tests passed!")

326 return 0

327 else

328 print("⚠️ Some tests failed - validation engine needs attention")

329 return 1

330 end