src/test-report-generator.lua
1#!/usr/bin/env lua
2
3-- Test script for report generator
4-- Tests all report formats and comparative analysis functionality
5
6local DIR = "/mnt/mtwo/programming/ai-stuff/neocities-modernization"
7
8package.path = package.path .. ';' .. DIR .. '/?.lua;' .. DIR .. '/libs/?.lua'
9
10local report_module = require("src.report-generator")
11local ReportGenerator = report_module.ReportGenerator
12local utils = require("libs.utils")
13
14-- {{{ function create_mock_validation_result
15function create_mock_validation_result(algorithm, accuracy_rate, comparisons_per_second)
16 return {
17 algorithm = algorithm or "cosine",
18 timestamp = os.date("%Y-%m-%d %H:%M:%S"),
19 duration_seconds = 15,
20 statistics = {
21 total_comparisons = 1000,
22 accurate_scores = math.floor((accuracy_rate or 0.95) * 1000),
23 inaccurate_scores = math.floor((1 - (accuracy_rate or 0.95)) * 1000),
24 missing_embeddings = 2,
25 accuracy_rate = accuracy_rate or 0.95,
26 error_rate = 1 - (accuracy_rate or 0.95),
27 tolerance = 0.001
28 },
29 performance = {
30 comparisons_per_second = comparisons_per_second or 67.5,
31 avg_comparison_time_ms = 1000 / (comparisons_per_second or 67.5)
32 },
33 discrepancies = {
34 count = math.floor((1 - (accuracy_rate or 0.95)) * 1000),
35 samples = {
36 {
37 poem_a = 123,
38 poem_b = 456,
39 stored_score = 0.85,
40 calculated_score = 0.82,
41 difference = 0.03,
42 relative_error = 0.035
43 },
44 {
45 poem_a = 789,
46 poem_b = 101,
47 stored_score = 0.42,
48 calculated_score = 0.45,
49 difference = 0.03,
50 relative_error = 0.071
51 }
52 },
53 max_difference = 0.05,
54 avg_difference = 0.025
55 },
56 errors = {
57 count = 2,
58 by_type = {
59 missing_embedding = 2
60 },
61 samples = {
62 {
63 type = "missing_embedding",
64 poem_a = 999,
65 poem_b = 888,
66 missing = "poem_a"
67 }
68 }
69 },
70 recommendations = {
71 "Excellent accuracy rate. Stored similarity data appears reliable.",
72 "2 missing embeddings found. Update embedding data or clean similarity matrix."
73 }
74 }
75end
76-- }}}
77
78-- {{{ function test_html_report_generation
79function test_html_report_generation()
80 print("š Testing HTML Report Generation")
81 print("================================")
82
83 local generator = ReportGenerator:new({format = "html"})
84 local mock_result = create_mock_validation_result("cosine", 0.95, 67.5)
85
86 local output_file = DIR .. "/test_html_report.html"
87
88 local success, result = pcall(function()
89 return generator:generate_validation_report(mock_result, output_file)
90 end)
91
92 if success and utils.file_exists(output_file) then
93 -- Check that HTML file contains expected content
94 local content = utils.read_file(output_file)
95
96 local checks = {
97 content:match("<!DOCTYPE html>") and "HTML doctype",
98 content:match("Similarity Validation Report") and "Title",
99 content:match("cosine") and "Algorithm name",
100 content:match("95%.0%%") and "Accuracy percentage",
101 content:match("1000") and "Total comparisons",
102 content:match("67%.5") and "Performance metrics"
103 }
104
105 local passed_checks = 0
106 for _, check in ipairs(checks) do
107 if check then
108 passed_checks = passed_checks + 1
109 end
110 end
111
112 -- Clean up
113 os.remove(output_file)
114
115 if passed_checks == #checks then
116 print("ā
HTML report generation: PASSED")
117 print(string.format(" - All %d content checks passed", #checks))
118 return true
119 else
120 print(string.format("ā HTML report generation: FAILED - %d/%d content checks passed", passed_checks, #checks))
121 return false
122 end
123 else
124 print(string.format("ā HTML report generation: FAILED - %s", result or "file not created"))
125 return false
126 end
127end
128-- }}}
129
130-- {{{ function test_markdown_report_generation
131function test_markdown_report_generation()
132 print("\nš Testing Markdown Report Generation")
133 print("====================================")
134
135 local generator = ReportGenerator:new({format = "markdown"})
136 local mock_result = create_mock_validation_result("euclidean", 0.88, 45.2)
137
138 local output_file = DIR .. "/test_markdown_report.md"
139
140 local success, result = pcall(function()
141 return generator:generate_validation_report(mock_result, output_file)
142 end)
143
144 if success and utils.file_exists(output_file) then
145 local content = utils.read_file(output_file)
146
147 local checks = {
148 content:match("# š Similarity Validation Report") and "Markdown header",
149 content:match("%*%*Algorithm:%*%* euclidean") and "Algorithm info",
150 content:match("88%.0%%") and "Accuracy percentage",
151 content:match("## š Validation Overview") and "Overview section",
152 content:match("## ā” Performance Metrics") and "Performance section",
153 content:match("```") and "Code blocks for performance data"
154 }
155
156 local passed_checks = 0
157 for _, check in ipairs(checks) do
158 if check then
159 passed_checks = passed_checks + 1
160 end
161 end
162
163 -- Clean up
164 os.remove(output_file)
165
166 if passed_checks == #checks then
167 print("ā
Markdown report generation: PASSED")
168 print(string.format(" - All %d content checks passed", #checks))
169 return true
170 else
171 print(string.format("ā Markdown report generation: FAILED - %d/%d content checks passed", passed_checks, #checks))
172 return false
173 end
174 else
175 print(string.format("ā Markdown report generation: FAILED - %s", result or "file not created"))
176 return false
177 end
178end
179-- }}}
180
181-- {{{ function test_json_report_generation
182function test_json_report_generation()
183 print("\nš§ Testing JSON Report Generation")
184 print("=================================")
185
186 local generator = ReportGenerator:new({format = "json"})
187 local mock_result = create_mock_validation_result("angular", 0.92, 55.8)
188
189 local output_file = DIR .. "/test_json_report.json"
190
191 local success, result = pcall(function()
192 return generator:generate_validation_report(mock_result, output_file)
193 end)
194
195 if success and utils.file_exists(output_file) then
196 -- Try to parse the JSON to verify it's valid
197 local json_data = utils.read_json_file(output_file)
198
199 if json_data and json_data.algorithm == "angular" and
200 json_data.statistics and json_data.statistics.accuracy_rate == 0.92 then
201 print("ā
JSON report generation: PASSED")
202 print(" - Valid JSON structure")
203 print(" - Contains expected data")
204
205 -- Clean up
206 os.remove(output_file)
207 return true
208 else
209 print("ā JSON report generation: FAILED - invalid JSON structure or data")
210 os.remove(output_file)
211 return false
212 end
213 else
214 print(string.format("ā JSON report generation: FAILED - %s", result or "file not created"))
215 return false
216 end
217end
218-- }}}
219
220-- {{{ function test_comparative_report_generation
221function test_comparative_report_generation()
222 print("\nš Testing Comparative Report Generation")
223 print("========================================")
224
225 -- Create multiple mock validation results
226 local validation_results = {
227 create_mock_validation_result("cosine", 0.95, 67.5),
228 create_mock_validation_result("euclidean", 0.88, 45.2),
229 create_mock_validation_result("angular", 0.92, 55.8),
230 create_mock_validation_result("manhattan", 0.85, 72.1)
231 }
232
233 local generator = ReportGenerator:new({format = "markdown"})
234 local output_file = DIR .. "/test_comparative_report.md"
235
236 local success, result = pcall(function()
237 return generator:generate_comparative_report(validation_results, output_file)
238 end)
239
240 if success and utils.file_exists(output_file) then
241 local content = utils.read_file(output_file)
242
243 local checks = {
244 content:match("# š Comparative Algorithm Validation Report") and "Comparative header",
245 content:match("cosine") and "Contains cosine algorithm",
246 content:match("euclidean") and "Contains euclidean algorithm",
247 content:match("95%.0%%") and "Contains cosine accuracy",
248 content:match("## š Algorithm Rankings") and "Rankings section",
249 content:match("## š” Recommendations") and "Recommendations section"
250 }
251
252 local passed_checks = 0
253 for _, check in ipairs(checks) do
254 if check then
255 passed_checks = passed_checks + 1
256 end
257 end
258
259 -- Clean up
260 os.remove(output_file)
261
262 if passed_checks == #checks then
263 print("ā
Comparative report generation: PASSED")
264 print(string.format(" - All %d content checks passed", #checks))
265 print(" - Successfully compared 4 algorithms")
266 return true
267 else
268 print(string.format("ā Comparative report generation: FAILED - %d/%d content checks passed", passed_checks, #checks))
269 return false
270 end
271 else
272 print(string.format("ā Comparative report generation: FAILED - %s", result or "file not created"))
273 return false
274 end
275end
276-- }}}
277
278-- {{{ function test_report_customization
279function test_report_customization()
280 print("\nāļø Testing Report Customization Options")
281 print("=======================================")
282
283 local mock_result = create_mock_validation_result("pearson_correlation", 0.78, 33.4)
284
285 -- Test with details disabled
286 local generator_no_details = ReportGenerator:new({
287 format = "html",
288 include_details = false,
289 include_recommendations = false
290 })
291
292 local output_file = DIR .. "/test_no_details_report.html"
293
294 local success, result = pcall(function()
295 return generator_no_details:generate_validation_report(mock_result, output_file)
296 end)
297
298 if success and utils.file_exists(output_file) then
299 local content = utils.read_file(output_file)
300
301 -- Should NOT contain discrepancy details or recommendations
302 local no_discrepancies = not content:match("Worst Discrepancies")
303 local no_recommendations = not content:match("Recommendations")
304 local has_basic_stats = content:match("78%.0%%") -- Should still have basic stats
305
306 -- Clean up
307 os.remove(output_file)
308
309 if no_discrepancies and no_recommendations and has_basic_stats then
310 print("ā
Report customization: PASSED")
311 print(" - Successfully excluded details and recommendations")
312 print(" - Maintained basic statistics")
313 return true
314 else
315 print("ā Report customization: FAILED")
316 print(string.format(" - No discrepancies: %s", no_discrepancies and "ā" or "ā"))
317 print(string.format(" - No recommendations: %s", no_recommendations and "ā" or "ā"))
318 print(string.format(" - Has basic stats: %s", has_basic_stats and "ā" or "ā"))
319 return false
320 end
321 else
322 print(string.format("ā Report customization: FAILED - %s", result or "file not created"))
323 return false
324 end
325end
326-- }}}
327
328-- {{{ function test_error_handling
329function test_error_handling()
330 print("\nšØ Testing Report Generator Error Handling")
331 print("==========================================")
332
333 local tests_passed = 0
334 local total_tests = 3
335
336 -- Test 1: Invalid format
337 print("Test 1: Invalid format")
338 local generator_bad_format = ReportGenerator:new({format = "invalid_format"})
339 local mock_result = create_mock_validation_result()
340
341 local success, error = pcall(function()
342 generator_bad_format:generate_validation_report(mock_result, DIR .. "/tmp/test.txt")
343 end)
344
345 if not success and error:match("Unsupported report format") then
346 print(" ā
Correctly detected invalid format")
347 tests_passed = tests_passed + 1
348 else
349 print(" ā Failed to detect invalid format")
350 end
351
352 -- Test 2: Missing validation result data
353 print("Test 2: Missing validation result data")
354 local generator = ReportGenerator:new({format = "html"})
355
356 success, error = pcall(function()
357 generator:generate_validation_report({}, DIR .. "/test_empty_result.html")
358 end)
359
360 if success then -- Should handle gracefully with defaults
361 print(" ā
Gracefully handled missing data")
362 tests_passed = tests_passed + 1
363 -- Clean up if file was created
364 if utils.file_exists(DIR .. "/test_empty_result.html") then
365 os.remove(DIR .. "/test_empty_result.html")
366 end
367 else
368 print(" ā Failed to handle missing data gracefully")
369 end
370
371 -- Test 3: Invalid output path
372 print("Test 3: Invalid output path")
373 success, error = pcall(function()
374 generator:generate_validation_report(mock_result, "/nonexistent/directory/report.html")
375 end)
376
377 if not success then
378 print(" ā
Correctly detected invalid output path")
379 tests_passed = tests_passed + 1
380 else
381 print(" ā Failed to detect invalid output path")
382 end
383
384 print(string.format("Error handling tests: %d/%d passed", tests_passed, total_tests))
385 return tests_passed == total_tests
386end
387-- }}}
388
389-- {{{ function test_template_substitution
390function test_template_substitution()
391 print("\nš§ Testing Template Variable Substitution")
392 print("=========================================")
393
394 local generator = ReportGenerator:new({format = "html"})
395 local mock_result = create_mock_validation_result("dot_product", 0.93, 41.7)
396
397 -- Test template substitution directly
398 local template = "Algorithm: {ALGORITHM}, Accuracy: {ACCURACY_PERCENT}%, Speed: {COMPARISONS_PER_SEC}"
399 local substituted = generator:substitute_template_vars(template, mock_result)
400
401 local expected = "Algorithm: dot_product, Accuracy: 93.0%, Speed: 41.7"
402
403 if substituted == expected then
404 print("ā
Template substitution: PASSED")
405 print(string.format(" - Input: %s", template))
406 print(string.format(" - Output: %s", substituted))
407 return true
408 else
409 print("ā Template substitution: FAILED")
410 print(string.format(" - Expected: %s", expected))
411 print(string.format(" - Got: %s", substituted))
412 return false
413 end
414end
415-- }}}
416
417-- {{{ function main
418function main()
419 print("š Report Generator Test Suite")
420 print("==============================\n")
421
422 local tests = {
423 {"HTML Report Generation", test_html_report_generation},
424 {"Markdown Report Generation", test_markdown_report_generation},
425 {"JSON Report Generation", test_json_report_generation},
426 {"Comparative Report Generation", test_comparative_report_generation},
427 {"Report Customization", test_report_customization},
428 {"Template Substitution", test_template_substitution},
429 {"Error Handling", test_error_handling}
430 }
431
432 local passed_tests = 0
433 local total_tests = #tests
434
435 for i, test in ipairs(tests) do
436 local test_name, test_func = test[1], test[2]
437
438 local success, result = pcall(test_func)
439
440 if success and result then
441 passed_tests = passed_tests + 1
442 print(string.format("\nā
%s: PASSED", test_name))
443 else
444 print(string.format("\nā %s: FAILED - %s", test_name, result or "unknown error"))
445 end
446 end
447
448 print(string.format("\n\nš Final Results: %d/%d tests passed", passed_tests, total_tests))
449
450 if passed_tests == total_tests then
451 print("š All report generator tests passed!")
452 return 0
453 else
454 print("ā ļø Some tests failed - report generator needs attention")
455 return 1
456 end
457end
458-- }}}
459
460-- Run tests if executed directly
461if arg and arg[0] and arg[0]:match("test%-report%-generator%.lua$") then
462 os.exit(main())
463end
464
465return {
466 test_html_report_generation = test_html_report_generation,
467 test_markdown_report_generation = test_markdown_report_generation,
468 test_json_report_generation = test_json_report_generation,
469 test_comparative_report_generation = test_comparative_report_generation,
470 test_report_customization = test_report_customization,
471 test_template_substitution = test_template_substitution,
472 test_error_handling = test_error_handling,
473 main = main,
474 create_mock_validation_result = create_mock_validation_result
475}