src/mass-diversity-generator.lua

22-- {{{ function escape_html

23local function escape_html(text)

24 if not text then return "" end

25 text = tostring(text)

26 text = text:gsub("&", "&")

27 text = text:gsub("<", "<")

28 text = text:gsub(">", ">")

29 text = text:gsub('"', """)

30 text = text:gsub("'", "'")

31 return text

32end

33-- }}}

35-- {{{ function ensure_directory

36local function ensure_directory(path)

37 -- Simple shell escaping for paths

38 local escaped_path = path:gsub("'", "'\"'\"'")

39 local success = os.execute("mkdir -p '" .. escaped_path .. "'")

40 return success == 0 or success == true

41end

42-- }}}

44-- {{{ function M.generate_diversity_chain_page

45function M.generate_diversity_chain_page(starting_poem_id, starting_poem, chain_result, output_dir, poems_data)

46 local template = [[<!DOCTYPE html>

47<html lang="en">

48<head>

49 <meta charset="UTF-8">

50 <meta name="viewport" content="width=device-width, initial-scale=1.0">

51 <title>Diversity Chain: {STARTING_TITLE}</title>

52 <style>

53 /* Base styles */

54 body {

55 font-family: Georgia, serif;

56 line-height: 1.6;

57 margin: 0;

58 padding: 1rem;

59 max-width: 1000px;

60 margin: 0 auto;

61 background: #fafafa;

62 }

63

64 .breadcrumb {

65 font-size: 0.9rem;

66 margin-bottom: 1rem;

67 color: #666;

68 }

69

70 .breadcrumb a {

71 color: #8b4513;

72 text-decoration: none;

73 }

74

75 .breadcrumb a:hover {

76 text-decoration: underline;

77 }

78

79 h1 {

80 color: #8b4513;

81 border-bottom: 2px solid #8b4513;

82 padding-bottom: 0.5rem;

83 }

84

85 .chain-description {

86 background: #f0f8ff;

87 padding: 1rem;

88 border-left: 4px solid #4682b4;

89 margin: 1rem 0;

90 font-style: italic;

91 }

92

93 .chain-stats {

94 display: flex;

95 gap: 2rem;

96 margin: 1rem 0;

97 font-size: 0.9rem;

98 color: #666;

99 }

100

101 .diversity-chain {

102 margin: 2rem 0;

103 }

104

105 .chain-poem {

106 border-left: 3px solid #8b4513;

107 margin: 1rem 0;

108 padding: 1rem;

109 background: #ffffff;

110 box-shadow: 0 1px 3px rgba(0,0,0,0.1);

111 transition: all 0.3s ease;

112 }

113

114 .chain-poem:hover {

115 background: #f9f9f9;

116 box-shadow: 0 2px 6px rgba(0,0,0,0.15);

117 }

118

119 .poem-number {

120 font-weight: bold;

121 color: #8b4513;

122 font-size: 0.9rem;

123 margin-bottom: 0.5rem;

124 }

125

126 .poem-title {

127 color: #2c5530;

128 margin: 0.5rem 0;

129 }

130

131 .poem-title a {

132 color: inherit;

133 text-decoration: none;

134 }

135

136 .poem-title a:hover {

137 text-decoration: underline;

138 }

139

140 .poem-excerpt {

141 color: #444;

142 margin: 0.5rem 0;

143 font-style: italic;

144 }

145

146 .diversity-score {

147 color: #666;

148 font-size: 0.8rem;

149 margin-top: 0.5rem;

150 padding: 0.25rem 0.5rem;

151 background: #f5f5f5;

152 border-radius: 3px;

153 display: inline-block;

154 }

155

156 .chain-actions {

157 margin: 2rem 0;

158 padding: 1rem;

159 background: #f8f8f8;

160 border-radius: 5px;

161 }

162

163 .chain-actions a {

164 display: inline-block;

165 margin-right: 1rem;

166 padding: 0.5rem 1rem;

167 background: #8b4513;

168 color: white;

169 text-decoration: none;

170 border-radius: 3px;

171 font-size: 0.9rem;

172 }

173

174 .chain-actions a:hover {

175 background: #5d2e0a;

176 }

177

178 @media (max-width: 600px) {

179 body {

180 padding: 0.5rem;

181 }

182

183 .chain-stats {

184 flex-direction: column;

185 gap: 0.5rem;

186 }

187

188 .chain-actions a {

189 display: block;

190 margin: 0.5rem 0;

191 text-align: center;

192 }

193 }

194 </style>

195</head>

196<body bgcolor="#000000" text="#FFFFFF" link="#6699FF" vlink="#9966FF">

197 <nav class="breadcrumb">

198 <a href="../../../index.html">Poetry Collection</a> →

199 <a href="../../index.html">Poems</a> →

200 <a href="../index.html">Diversity Chains</a> →

201 <span>{STARTING_TITLE}</span>

202 </nav>

203

204 <h1>🎭 Diversity Chain: {STARTING_TITLE}</h1>

205

206 <div class="chain-description">

207 A maximum diversity exploration starting from "{STARTING_TITLE}" –

208 each step leads to the <strong>least similar</strong> poem, creating unexpected literary connections

209 and challenging transitions that reveal hidden relationships across different themes and styles.

210 </div>

211

212 <div class="chain-stats">

213 <div><strong>Chain Length:</strong> {CHAIN_LENGTH} poems</div>

214 <div><strong>Average Diversity:</strong> {AVERAGE_DIVERSITY}</div>

215 <div><strong>Category:</strong> {CATEGORY}</div>

216 <div><strong>Generated:</strong> {GENERATION_DATE}</div>

217 </div>

218

219 <div class="diversity-chain">

220 {CHAIN_POEMS}

221 </div>

222

223 <div class="chain-actions">

224 <a href="../by-category/{CATEGORY}/">More {CATEGORY} chains</a>

225 <a href="../random-start/">Random diversity chain</a>

226 <a href="../browse/">Browse all chains</a>

227 </div>

228</body>

229</html>]]

230

231 local chain = chain_result.chain or {}

232 local metadata = chain_result.metadata or {}

233

234 -- Build chain poems HTML

235 local chain_html = ""

236 for i, poem_id in ipairs(chain) do

237 local poem = poems_data.poems and poems_data.poems[poem_id]

238 if poem then

239 local diversity_score = ""

240 if i > 1 and metadata.diversities and metadata.diversities[i-1] then

241 diversity_score = string.format('<div class="diversity-score">Diversity: %.2f</div>',

242 metadata.diversities[i-1])

243 end

244

245 chain_html = chain_html .. string.format([[

246<div class="chain-poem">

247 <div class="poem-number">Step %d</div>

248 <h3 class="poem-title"><a href="../../%s/poem-%03d.html">%s</a></h3>

249 <div class="poem-excerpt">%s</div>

250 %s

251</div>]],

252 i,

253 poem.category or "fediverse",

254 poem_id,

255 escape_html(poem.title or ("Poem " .. poem_id)),

256 escape_html(string.sub(poem.content or poem.text or "", 1, 120) .. (string.len(poem.content or poem.text or "") > 120 and "..." or "")),

257 diversity_score

258 )

259 end

260 end

261

262 -- Calculate average diversity display

263 local avg_diversity = metadata.average_diversity or 0

264 local avg_diversity_display = string.format("%.2f", avg_diversity)

265

266 -- Substitute template variables

267 local starting_title = escape_html(starting_poem.title or ("Poem " .. starting_poem_id))

268 local category = starting_poem.category or "fediverse"

269

270 template = template:gsub("{STARTING_TITLE}", starting_title)

271 template = template:gsub("{CATEGORY}", category)

272 template = template:gsub("{CHAIN_LENGTH}", tostring(#chain))

273 template = template:gsub("{AVERAGE_DIVERSITY}", avg_diversity_display)

274 template = template:gsub("{GENERATION_DATE}", os.date("%Y-%m-%d"))

275 template = template:gsub("{CHAIN_POEMS}", chain_html)

276

277 -- Ensure output directory exists

278 local category_dir = string.format("%s/poems/diversity/by-category/%s",

279 output_dir, category)

280 if not ensure_directory(category_dir) then

281 utils.log_error("Failed to create directory: " .. category_dir)

282 return nil

283 end

284

285 -- Write page

286 local output_file = string.format("%s/poem-%03d.html", category_dir, starting_poem_id)

287 local success = utils.write_file(output_file, template)

288

289 if success then

290 return {

291 file = output_file,

292 starting_poem = starting_poem_id,

293 chain_length = #chain,

294 category = category,

295 average_diversity = avg_diversity

296 }

297 else

298 utils.log_error("Failed to write diversity page: " .. output_file)

299 return nil

300 end

301end

302-- }}}

304-- {{{ function M.generate_diversity_batch

305function M.generate_diversity_batch(poems_batch, similarity_data, output_dir, poems_data, config)

306 local results = {}

307 local failed_count = 0

308

309 for _, poem_entry in ipairs(poems_batch) do

310 local poem_id = poem_entry.id

311 local poem_data = poem_entry.data

312

313 -- Generate diversity chain

314 local diversity_chain_result = diversity.generate_maximum_diversity_chain(

315 poem_id,

316 poems_data.poems, -- Pass the poems table, not the full poems_data object

317 similarity_data,

318 config

319 )

320

321 if diversity_chain_result and diversity_chain_result.chain and #diversity_chain_result.chain >= 1 then

322 -- Generate HTML page

323 local page_result = M.generate_diversity_chain_page(

324 poem_id,

325 poem_data,

326 diversity_chain_result,

327 output_dir,

328 poems_data

329 )

330

331 if page_result then

332 table.insert(results, page_result)

333 else

334 failed_count = failed_count + 1

335 utils.log_warn(string.format("Failed to generate HTML page for poem %s", poem_id))

336 end

337 else

338 failed_count = failed_count + 1

339 utils.log_warn(string.format("Failed to generate diversity chain for poem %s", poem_id))

340 end

341 end

342

343 if failed_count > 0 then

344 utils.log_warn(string.format("Batch completed with %d failures out of %d poems",

345 failed_count, #poems_batch))

346 end

347

348 return results

349end

350-- }}}

352-- {{{ function M.generate_all_diversity_chain_pages

353function M.generate_all_diversity_chain_pages(poems_data, similarity_data, output_dir, config)

354 config = config or diversity.DiversityConfig:new({

355 chain_length = 15,

356 debug_logging = false

357 })

358

359 local batch_size = 50 -- Manageable batch size for memory efficiency

360

361 -- Count total poems

362 local total_poems = 0

363 local poem_list = {}

364 for poem_id, poem_data in pairs(poems_data.poems or {}) do

365 total_poems = total_poems + 1

366 table.insert(poem_list, {id = tonumber(poem_id), data = poem_data})

367 end

368

369 -- Sort by ID for consistent processing

370 table.sort(poem_list, function(a, b) return a.id < b.id end)

371

372 utils.log_info(string.format("🏭 Generating diversity pages for %d poems...", total_poems))

373 utils.log_info(string.format("📊 Target chain length: %d, Batch size: %d",

374 config.chain_length, batch_size))

375

376 local generated_pages = {}

377 local batch_count = 0

378 local current_batch = {}

379 local start_time = os.clock()

380

381 for _, poem_entry in ipairs(poem_list) do

382 table.insert(current_batch, poem_entry)

383

384 if #current_batch >= batch_size then

385 local batch_results = M.generate_diversity_batch(

386 current_batch, similarity_data, output_dir, poems_data, config

387 )

388

389 for _, result in ipairs(batch_results) do

390 table.insert(generated_pages, result)

391 end

392

393 batch_count = batch_count + 1

394 current_batch = {}

395

396 local elapsed = os.clock() - start_time

397 local rate = #generated_pages / elapsed

398 local eta = (total_poems - #generated_pages) / rate

399

400 utils.log_info(string.format("📈 Batch %d complete: %d/%d pages (%.1f%%, %.1f pages/sec, ETA: %.0f min)",

401 batch_count, #generated_pages, total_poems,

402 (#generated_pages / total_poems) * 100,

403 rate, eta / 60))

404

405 -- Memory cleanup hint

406 collectgarbage()

407 end

408 end

409

410 -- Process final batch

411 if #current_batch > 0 then

412 local batch_results = M.generate_diversity_batch(

413 current_batch, similarity_data, output_dir, poems_data, config

414 )

415 for _, result in ipairs(batch_results) do

416 table.insert(generated_pages, result)

417 end

418 batch_count = batch_count + 1

419 end

420

421 local total_time = os.clock() - start_time

422

423 utils.log_info(string.format("✅ Mass generation complete: %d pages in %.1f seconds (%.1f pages/sec)",

424 #generated_pages, total_time, #generated_pages / total_time))

425

426 return {

427 total_pages = #generated_pages,

428 requested_pages = total_poems,

429 success_rate = #generated_pages / total_poems,

430 output_directory = output_dir .. "/poems/diversity",

431 generation_time_seconds = total_time,

432 generation_rate = #generated_pages / total_time,

433 pages = generated_pages,

434 batch_count = batch_count,

435 config = config

436 }

437end

438-- }}}

440-- {{{ function M.generate_index_pages

441function M.generate_index_pages(generation_result, output_dir)

442 -- Create main diversity index page

443 local diversity_dir = output_dir .. "/poems/diversity"

444 ensure_directory(diversity_dir)

445

446 local index_template = [[<!DOCTYPE html>

447<html lang="en">

448<head>

449 <meta charset="UTF-8">

450 <meta name="viewport" content="width=device-width, initial-scale=1.0">

451 <title>Diversity Chain Explorer</title>

452 <style>

453 body {

454 font-family: Georgia, serif;

455 line-height: 1.6;

456 margin: 0;

457 padding: 1rem;

458 max-width: 1000px;

459 margin: 0 auto;

460 background: #fafafa;

461 }

462

463 h1 {

464 color: #8b4513;

465 border-bottom: 2px solid #8b4513;

466 padding-bottom: 0.5rem;

467 }

468

469 .description {

470 background: #f0f8ff;

471 padding: 1rem;

472 border-left: 4px solid #4682b4;

473 margin: 1rem 0;

474 }

475

476 .stats {

477 display: grid;

478 grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));

479 gap: 1rem;

480 margin: 2rem 0;

481 }

482

483 .stat-card {

484 background: white;

485 padding: 1rem;

486 border-radius: 5px;

487 box-shadow: 0 1px 3px rgba(0,0,0,0.1);

488 }

489

490 .stat-number {

491 font-size: 1.5rem;

492 font-weight: bold;

493 color: #8b4513;

494 }

495

496 .navigation {

497 display: grid;

498 grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));

499 gap: 1rem;

500 margin: 2rem 0;

501 }

502

503 .nav-card {

504 background: white;

505 padding: 1.5rem;

506 border-radius: 5px;

507 box-shadow: 0 2px 6px rgba(0,0,0,0.1);

508 text-align: center;

509 border: 2px solid transparent;

510 transition: all 0.3s ease;

511 }

512

513 .nav-card:hover {

514 border-color: #8b4513;

515 transform: translateY(-2px);

516 }

517

518 .nav-card h3 {

519 color: #8b4513;

520 margin-top: 0;

521 }

522

523 .nav-card a {

524 text-decoration: none;

525 color: inherit;

526 }

527 </style>

528</head>

529<body bgcolor="#000000" text="#FFFFFF" link="#6699FF" vlink="#9966FF">

530 <h1>🎭 Diversity Chain Explorer</h1>

531

532 <div class="description">

533 <p><strong>Diversity chains</strong> create unexpected literary journeys by connecting poems through

534 <em>maximum dissimilarity</em>. Each step leads to the least similar poem, creating surprising

535 transitions and revealing hidden connections across different themes, styles, and emotions.</p>

536 </div>

537

538 <div class="stats">

539 <div class="stat-card">

540 <div class="stat-number">{TOTAL_PAGES}</div>

541 <div>Diversity Chains</div>

542 </div>

543 <div class="stat-card">

544 <div class="stat-number">{SUCCESS_RATE}%</div>

545 <div>Generation Success</div>

546 </div>

547 <div class="stat-card">

548 <div class="stat-number">{GENERATION_TIME}s</div>

549 <div>Generation Time</div>

550 </div>

551 <div class="stat-card">

552 <div class="stat-number">{GENERATION_RATE}</div>

553 <div>Pages/Second</div>

554 </div>

555 </div>

556

557 <div class="navigation">

558 <div class="nav-card">

559 <a href="by-category/">

560 <h3>📚 Browse by Category</h3>

561 <p>Explore diversity chains organized by poem categories like fediverse, messages, and notes.</p>

562 </a>

563 </div>

564

565 <div class="nav-card">

566 <a href="random-start/">

567 <h3>🎲 Random Start</h3>

568 <p>Begin a diversity journey from a randomly selected poem for serendipitous discovery.</p>

569 </a>

570 </div>

571

572 <div class="nav-card">

573 <a href="browse/">

574 <h3>🗂️ Browse All</h3>

575 <p>View complete alphabetical listing of all available diversity chain starting points.</p>

576 </a>

577 </div>

578 </div>

579</body>

580</html>]]

581

582 -- Substitute stats

583 index_template = index_template:gsub("{TOTAL_PAGES}", tostring(generation_result.total_pages))

584 index_template = index_template:gsub("{SUCCESS_RATE}", string.format("%.1f", generation_result.success_rate * 100))

585 index_template = index_template:gsub("{GENERATION_TIME}", string.format("%.1f", generation_result.generation_time_seconds))

586 index_template = index_template:gsub("{GENERATION_RATE}", string.format("%.1f", generation_result.generation_rate))

587

588 -- Write index page

589 local index_file = diversity_dir .. "/index.html"

590 utils.write_file(index_file, index_template)

591

592 utils.log_info("📄 Created diversity index page: " .. index_file)

593

594 return {

595 index_page = index_file

596 }

597end

598-- }}}

600-- {{{ function M.test_mass_generation

601function M.test_mass_generation(similarity_file, poems_file, output_dir, test_count)

602 test_count = test_count or 10

603

604 utils.log_info("🧪 Testing mass diversity generation with " .. test_count .. " poems")

605

606 -- Load data

607 local similarity_data = diversity.load_similarity_data(similarity_file)

608 if not similarity_data then return false end

609

610 local poems_data = utils.read_json_file(poems_file)

611 if not poems_data or not poems_data.poems then

612 utils.log_error("Failed to load poems data")

613 return false

614 end

615

616 -- Create limited dataset for testing

617 local test_poems = {poems = {}}

618 local count = 0

619 for poem_id, poem_data in pairs(poems_data.poems) do

620 if count >= test_count then break end

621 test_poems.poems[poem_id] = poem_data

622 count = count + 1

623 end

624

625 utils.log_info(string.format("Test dataset created with %d poems", count))

626

627 -- Configure test

628 local config = diversity.DiversityConfig:new({

629 chain_length = 5, -- Shorter chains for testing

630 debug_logging = false

631 })

632

633 -- Generate test pages

634 local result = M.generate_all_diversity_chain_pages(test_poems, similarity_data, output_dir, config)

635

636 if result then

637 utils.log_info(string.format("✅ Test successful: %d/%d pages generated (%.1f%% success rate)",

638 result.total_pages, result.requested_pages, result.success_rate * 100))

639

640 -- Generate index pages

641 M.generate_index_pages(result, output_dir)

642

643 return result

644 else

645 utils.log_error("❌ Test failed")

646 return false

647 end

648end

649-- }}}