scripts/extract-fediverse.lua

753 lines

2-- Fediverse content extraction script

3-- Parses ActivityPub JSON and extracts formatted posts with attachment metadata

4--

5-- ACTIVITYPUB ATTACHMENT FORMAT (Mastodon/W3C Standard):

6-- Each Note object in outbox.json may contain an "attachment" array:

7-- {

8-- "type": "Create",

9-- "object": {

10-- "type": "Note",

11-- "content": "Post text here",

12-- "attachment": [

13-- {

14-- "type": "Document",

15-- "mediaType": "image/png", -- MIME type (image/png, image/jpeg, video/mp4, etc.)

16-- "url": "https://server.com/media/files/123/456/789/original/abc123.png",

17-- "name": "Alt text description", -- User-provided alt text (may be null)

18-- "blurhash": "LEHV6nWB2yk8...", -- Blur hash for placeholder (optional)

19-- "width": 1920, -- Image dimensions (optional)

20-- "height": 1080

21-- }

22-- ]

23-- }

24-- }

25--

26-- URL PATH MAPPING:

27-- The URL path structure maps directly to local media_attachments directory:

28-- URL: https://tech.lgbt/media/files/113/464/378/730/595/557/original/658cbf8cc6804a09.png

29-- Local: input/media_attachments/files/113/464/378/730/595/557/original/658cbf8cc6804a09.png

30--

31-- The numeric segments (113/464/378/...) are derived from Mastodon's internal attachment ID

32-- split into 3-digit chunks for filesystem distribution.

34-- {{{ setup_dir_path

35local function setup_dir_path(provided_dir)

36 if provided_dir then

37 return provided_dir

38 end

39 return "/mnt/mtwo/programming/ai-stuff/neocities-modernization"

40end

41-- }}}

43-- {{{ parse_args

44-- Parse command line arguments for DIR, source override, and boost inclusion

45local function parse_args(args)

46 local dir = nil

47 local source_override = nil

48 local include_boosts = nil -- nil means use config default

49 local i = 1

51 while i <= #(args or {}) do

52 local a = args[i]

53 if a == "--include-boosts" then

54 include_boosts = true

55 i = i + 1

56 elseif a == "--no-boosts" then

57 include_boosts = false

58 i = i + 1

59 elseif not a:match("^%-") then

60 -- Positional arguments: first is DIR, second is source override

61 if not dir then

62 dir = a

63 else

64 source_override = a

65 end

66 i = i + 1

67 else

68 i = i + 1

69 end

70 end

72 return dir, source_override, include_boosts

73end

74-- }}}

76-- Get project directory and options from command line

77local parsed_dir, OVERRIDE_SOURCE, CLI_INCLUDE_BOOSTS = parse_args(arg)

78local DIR = setup_dir_path(parsed_dir)

80-- Set up package path to find libs

81package.path = DIR .. "/libs/?.lua;" .. package.path

82local dkjson = require("dkjson")

83local exclusion_filter = require("exclusion-filter")

84-- Issue 4-003 (August 2026): typed-text reconstruction and compose-box

85-- counting live in a shared, tested library instead of an inline cleaner

86local typed_text = require("mastodon-typed-text")

88-- Issue 10-003: Load unified config from config.lua

89local config_loader = require("config-loader")

90config_loader.set_project_root(DIR)

91local config = config_loader.load()

93-- Issue 10-015: Load sources configuration for multi-directory support

94local sources_loader = require("sources-loader")

95sources_loader.set_project_root(DIR)

97-- ANSI color codes for terminal output

98local COLOR_GREEN = "\027[92m" -- Bright green for success (✓, ✅)

99local COLOR_BLUE = "\027[94m" -- Bright blue for info (ℹ️)

100local COLOR_RED = "\027[91m" -- Bright red for errors (✗, ❌)

101local COLOR_YELLOW = "\027[93m" -- Bright yellow for warnings (⚠️)

102local COLOR_RESET = "\027[0m" -- Reset to default

103

104-- {{{ local function relative_path

105-- Issue 7-003: Show project name instead of "./" when path equals DIR

106local function relative_path(absolute_path)

107 if absolute_path == DIR or absolute_path == DIR .. "/" then

108 local dir_name = DIR:match("([^/]+)/?$")

109 return dir_name .. "/"

110 end

111 if absolute_path:sub(1, #DIR) == DIR then

112 local rel = absolute_path:sub(#DIR + 1)

113 if rel:sub(1, 1) == "/" then rel = rel:sub(2) end

114 return "./" .. rel

115 end

116 return absolute_path

117end

118-- }}}

119

120-- Issue 10-015a: Get fediverse path from unified sources config (no fallback - errors if not configured)

121local fediverse_directories = sources_loader.get_directories("fediverse")

122if #fediverse_directories == 0 then

123 print(COLOR_RED .. "❌ Error: sources.fediverse not configured in config.lua" .. COLOR_RESET)

124 os.exit(1)

125end

126-- Use the primary directory from sources config

127local fediverse_backup_path = fediverse_directories[1].path

128-- Strip DIR prefix if present (sources-loader returns absolute paths)

129if fediverse_backup_path:sub(1, #DIR) == DIR then

130 fediverse_backup_path = fediverse_backup_path:sub(#DIR + 2) -- +2 for the slash

131end

132

133-- Privacy configuration from unified config

134-- CLI flags --include-boosts/--no-boosts override config value

135local function get_include_boosts()

136 if CLI_INCLUDE_BOOSTS ~= nil then

137 return CLI_INCLUDE_BOOSTS

138 end

139 return config.privacy.include_boosts or false

140end

141

142-- {{{ local function load_boost_content_cache

143-- Load scraped boost content cache from assets/boost-content-cache.json

144-- Returns a table mapping URI -> cached content data

145local boost_content_cache = nil

146local function load_boost_content_cache()

147 if boost_content_cache then

148 return boost_content_cache

149 end

150

151 local cache_path = DIR .. "/assets/boost-content-cache.json"

152 local file = io.open(cache_path, "r")

153 if not file then

154 boost_content_cache = {}

155 return boost_content_cache

156 end

157

158 local content = file:read("*a")

159 file:close()

160

161 local data, pos, err = dkjson.decode(content)

162 if err or not data or not data.entries then

163 boost_content_cache = {}

164 return boost_content_cache

165 end

166

167 boost_content_cache = data.entries

168 local count = 0

169 for _ in pairs(boost_content_cache) do count = count + 1 end

170 print(" 📥 Loaded boost content cache: " .. count .. " entries")

171 return boost_content_cache

172end

173-- }}}

174

175local privacy_config = {

176 mode = config.privacy.mode or "clean",

177 anonymization_prefix = config.privacy.anonymization_prefix or "user-",

178 include_boosts = get_include_boosts(),

179 preserve_original_length = config.privacy.preserve_original_length or true,

180 store_anonymization_map = config.privacy.store_anonymization_map or false,

181 local_server_domain = config.privacy.local_server_domain or "tech.lgbt",

182 debug_anonymization = false -- Debug flag, not in config

183}

184

185-- Log boost inclusion status

186if privacy_config.include_boosts then

187 print("📤 Including fediverse boosts in extraction (CLI flag or config)")

188end

189

190-- Use override path if provided (for ZIP extraction), otherwise use configured path

191local source_base_path

192if OVERRIDE_SOURCE then

193 source_base_path = OVERRIDE_SOURCE

194 print("🔄 Using temporary extraction source: " .. relative_path(source_base_path))

195else

196 source_base_path = DIR .. "/" .. fediverse_backup_path

197 print("🔄 Using configured source: " .. relative_path(source_base_path))

198end

199

200-- Set up file paths - check if we're already in extract directory

201local file

202if source_base_path:match("extract$") then

203 file = source_base_path .. "/outbox.json"

204else

205 file = source_base_path .. "/extract/outbox.json"

206end

207local save_location = DIR .. "/" .. fediverse_backup_path .. "/files"

208

209-- Load and parse ActivityPub data

210print("🔄 Loading ActivityPub data from: " .. relative_path(file))

211local opened_file = io.open(file, "r")

212if not opened_file then

213 -- Issue 7-006: Full-line coloring for error messages

214 print(COLOR_RED .. "❌ Error: Could not open file " .. file .. COLOR_RESET)

215 print(" Make sure the file exists and is readable")

216 os.exit(1)

217end

218

219local opened_file_string = opened_file:read("*a")

220opened_file:close()

221

222local data = dkjson.decode(opened_file_string)

223if not data then

224 -- Issue 7-006: Full-line coloring for error messages

225 print(COLOR_RED .. "❌ Error: Could not parse JSON data from " .. file .. COLOR_RESET)

226 os.exit(1)

227end

228

229-- Issue 7-006: Full-line coloring for success messages

230print(COLOR_GREEN .. "✅ Loaded ActivityPub data: " .. (data.totalItems or #data.orderedItems) .. " activities" .. COLOR_RESET)

231

232-- Issue 6-031: Load poem exclusion filter

233-- Excluded poems leave gaps in the ID sequence (tombstoning) to preserve stable anchor links

234local poem_exclusions = exclusion_filter.load_default(DIR)

235if poem_exclusions:count() > 0 then

236 -- Issue 7-006: Full-line coloring for info messages

237 print(COLOR_YELLOW .. "🚫 Exclusion filter loaded: " .. poem_exclusions:summary() .. COLOR_RESET)

238end

239

240-- Privacy system variables

241local user_anonymization_map = {}

242local user_counter = 1

243

244-- {{{ function normalize_username

245local function normalize_username(username)

246 -- Strip ID paths and normalize username variations for consistent mapping

247 -- Remove paths like "/111978500472309702" from usernames

248 local normalized = username:gsub("/[0-9]+", "")

249

250 -- Handle specific username variations - map shorter forms to longer canonical forms

251 -- This is based on observed patterns in the fediverse data

252 local username_mappings = {

253 ["wyatt"] = "wyatt8740", -- Map @wyatt to @wyatt8740 for consistency

254 -- Add other mappings here as needed

255 }

256

257 -- Apply username mapping if one exists

258 if username_mappings[normalized] then

259 normalized = username_mappings[normalized]

260 end

261

262 return normalized

263end

264-- }}}

265

266-- {{{ function anonymize_mention

267local function anonymize_mention(username, server)

268 -- Normalize username to handle variations and ID paths

269 local normalized_username = normalize_username(username)

270

271 -- Debug logging to track anonymization mappings

272 if privacy_config.debug_anonymization then

273 io.stderr:write(string.format("DEBUG: anonymize_mention: '%s' -> '%s' @ '%s'\n",

274 username, normalized_username, server or "local"))

275 end

276

277 -- IMPORTANT: Consider users with same username on different servers as the same person

278 -- This handles server migrations and cross-server mentions of the same person

279 -- We only use the username for mapping, ignoring the server domain entirely

280 local map_key = normalized_username -- Just username, no server

281

282 if not user_anonymization_map[map_key] then

283 user_anonymization_map[map_key] = privacy_config.anonymization_prefix .. user_counter

284 user_counter = user_counter + 1

285 if privacy_config.debug_anonymization then

286 io.stderr:write(string.format(" -> New mapping: %s = %s\n", map_key, user_anonymization_map[map_key]))

287 end

288 end

289 return user_anonymization_map[map_key]

290end

291-- }}}

292

293-- {{{ function process_mentions_for_privacy

294local function process_mentions_for_privacy(content, privacy_mode)

295 if privacy_mode ~= "clean" then

296 return content, content -- Return original for dirty mode

297 end

298

299 local original_content = content

300 local processed_content = content

301

302 -- Handle HTML mention markup: ...<a href="https://server/@user">@user</a>

303 processed_content = processed_content:gsub(']*>.-<a href="[^"]*://([^/"]+)/@([^"/?"]*)[^"]*"[^>]*>@([^<]*)</a>', function(server, user, display_user)

304 -- Use the URL username (user) which is more reliable than display text

305 -- The URL contains the actual username, display might be shortened

306 -- Extract only the username part, not any path segments or IDs after it

307 return "@" .. anonymize_mention(user, server)

308 end)

309

310 -- Handle simpler HTML mentions: <a href="https://server/users/user" class="u-url mention">@user</a>

311 processed_content = processed_content:gsub('<a href="[^"]*://([^/"]+)/users/([^"/?"]*)[^"]*"[^>]*>@([^<]*)</a>', function(server, user, display_user)

312 -- Use the URL username (user) which is more reliable than display text

313 -- Extract only the username part, not any path segments after it

314 return "@" .. anonymize_mention(user, server)

315 end)

316

317 -- 6-027a Patterns: Handle plain text mentions as specified in sub-issue

318 -- Pattern 1: Full mentions @user@domain.com

319 processed_content = processed_content:gsub("@([%w%.%-_]+)@([%w%.%-]+%.%w+)", function(user, server)

320 return "@" .. anonymize_mention(user, server)

321 end)

322

323 -- Pattern 2: Multiple usernames at start - handle sequences like "@user1 @user2 @user3 content"

324 -- This pattern handles multiple consecutive mentions at the beginning

325 while processed_content:match("^@[%w%.%-_]+%s+@") do

326 processed_content = processed_content:gsub("^@([%w%.%-_]+)(%s+)", function(user, space)

327 return "@" .. anonymize_mention(user, nil) .. space

328 end)

329 end

330

331 -- Pattern 3: Single username at start (after multiple handling)

332 processed_content = processed_content:gsub("^@([%w%.%-_]+)%s", function(user)

333 return "@" .. anonymize_mention(user, nil) .. " "

334 end)

335

336 -- Pattern 4: Local mentions @user (same server, followed by whitespace)

337 processed_content = processed_content:gsub("@([%w%.%-_]+)%s", function(user)

338 return "@" .. anonymize_mention(user, nil) .. " "

339 end)

340

341 -- Pattern 5: @user at end of content (no trailing space)

342 processed_content = processed_content:gsub("@([%w%.%-_]+)$", function(user)

343 return "@" .. anonymize_mention(user, nil)

344 end)

345

346 -- Pattern 6: Catch any remaining @username patterns in the middle of text

347 -- This catches mentions followed by punctuation or other non-space characters

348 processed_content = processed_content:gsub("@([%w%.%-_]+)([^%w%.%-_@])", function(user, following_char)

349 return "@" .. anonymize_mention(user, nil) .. following_char

350 end)

351

352 return processed_content, original_content

353end

354-- }}}

355

356-- {{{ function categorize_activity

357local function categorize_activity(activity)

358 if activity.type == "Create" and activity.object and activity.object.type == "Note" then

359 return "original_post", activity.object

360 elseif activity.type == "Announce" then

361 return "boost", activity.object

362 else

363 return "unknown", nil

364 end

365end

366-- }}}

367

368-- {{{ function extract_boost_content

369local function extract_boost_content(announce_activity)

370 local boosted_object = announce_activity.object

371

372 -- If object is URI, check cache for scraped content first

373 if type(boosted_object) == "string" then

374 local cache = load_boost_content_cache()

375 local cached = cache[boosted_object]

376

377 -- Issue 10-037: Check for non-empty content (empty string "" is truthy in Lua)

378 -- Cache entries with empty content should fall back to "External post:" format

379 if cached and cached.content and cached.content ~= "" then

380 -- Use cached scraped content instead of placeholder

381 -- The cached content is HTML; typed-text restoration processes it later

382 return {

383 type = "cached_external_boost",

384 uri = boosted_object,

385 boost_timestamp = announce_activity.published,

386 content = cached.content,

387 content_warning = cached.summary, -- CW from original post

388 sensitive = cached.sensitive,

389 original_published = cached.published,

390 original_author = cached.attributed_to,

391 metadata = {

392 is_boost = true,

393 boost_type = "cached_external",

394 original_uri = boosted_object,

395 boost_date = announce_activity.published,

396 scraped_at = cached.scraped_at,

397 original_author = cached.attributed_to

398 }

399 }

400 end

401

402 -- No cache entry - fall back to placeholder

403 return {

404 type = "external_boost",

405 uri = boosted_object,

406 boost_timestamp = announce_activity.published,

407 content = "External post: " .. boosted_object,

408 metadata = {

409 is_boost = true,

410 boost_type = "external",

411 original_uri = boosted_object,

412 boost_date = announce_activity.published

413 }

414 }

415 end

416

417 -- If object is embedded, extract full content

418 -- Issue 10-037: Also check for non-empty embedded content

419 if type(boosted_object) == "table" and boosted_object.content and boosted_object.content ~= "" then

420 return {

421 type = "embedded_boost",

422 content = boosted_object.content,

423 original_author = boosted_object.attributedTo,

424 boost_timestamp = announce_activity.published,

425 original_timestamp = boosted_object.published,

426 metadata = {

427 is_boost = true,

428 boost_type = "embedded",

429 original_author = boosted_object.attributedTo,

430 boost_date = announce_activity.published,

431 original_date = boosted_object.published

432 }

433 }

434 end

435

436 -- Issue 10-037: Fallback for embedded objects with empty content

437 -- Extract URI from the object's id field and create placeholder entry

438 if type(boosted_object) == "table" and boosted_object.id then

439 return {

440 type = "external_boost",

441 uri = boosted_object.id,

442 boost_timestamp = announce_activity.published,

443 content = "External post: " .. boosted_object.id,

444 metadata = {

445 is_boost = true,

446 boost_type = "embedded_empty",

447 original_uri = boosted_object.id,

448 original_author = boosted_object.attributedTo,

449 boost_date = announce_activity.published,

450 content_unavailable = true

451 }

452 }

453 end

454

455 return nil

456end

457-- }}}

458

459-- HTML-to-text now lives in libs/mastodon-typed-text.lua (issue 4-003).

460-- The old inline clean_html stripped emphasis tags without restoring the

461-- markdown delimiters the author typed (love was *love* in the

462-- compose box), silently shaving 2-4 characters off hundreds of poems and

463-- disqualifying them from golden status. The library restores delimiters

464-- for display AND counts the way the compose box counted.

465

466-- {{{ function process_fediverse_content

467local function process_fediverse_content(raw_content, cw, privacy_mode)

468 if not raw_content then return nil end

469

470 -- Process mentions for privacy BEFORE HTML cleaning to preserve structure

471 local privacy_processed_content, original_content = process_mentions_for_privacy(raw_content, privacy_mode)

472

473 -- Reconstruct typed text for display content (after anonymization)

474 local clean_content = typed_text.restore(privacy_processed_content)

475

476 return {

477 content = clean_content,

478 raw_content = raw_content,

479 -- pre-anonymization HTML: golden counting must price the real

480 -- @mentions the author typed, not the anonymized replacements

481 original_content = original_content,

482 content_warning = (cw and cw ~= "") and cw or nil,

483 privacy_applied = (privacy_mode == "clean")

484 }

485end

486-- }}}

487

488-- {{{ function extract_date

489local function extract_date(timestamp)

490 return timestamp and timestamp:match("(%d%d%d%d%-%d%d%-%d%d)") or "0000-00-00"

491end

492-- }}}

493

494-- {{{ function extract_full_date

495local function extract_full_date(timestamp)

496 if timestamp then

497 return timestamp:match("(%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d)") or timestamp

498 end

499 return os.date("%Y-%m-%dT%H:%M:%S")

500end

501-- }}}

502

503-- {{{ function generate_poem_metadata

504local function generate_poem_metadata(content, cw, source_data, original_html)

505 -- Golden poem calculation (issue 4-003): count what the author watched in

506 -- the compose box, not what the archive stores. The library reconstructs

507 -- typed text from pre-anonymization HTML (markdown delimiters restored,

508 -- real @mentions priced at visible text, URLs at a flat 23), measures in

509 -- UTF-16 units like the compose box did, and adds the CW text without

510 -- any "CW: " prefix (per 6-027).

511 local golden_poem_length = typed_text.compose_box_count(original_html or content, cw)

512

513 local metadata = {

514 character_count = string.len(content), -- Display content length (post-privacy)

515 golden_poem_character_count = golden_poem_length, -- For golden poem qualification (1024 chars)

516 is_golden_poem = (golden_poem_length == 1024),

517 word_count = select(2, content:gsub("%S+", "")),

518 has_content_warning = (cw and cw ~= ""),

519 extraction_timestamp = os.date("%Y-%m-%dT%H:%M:%SZ")

520 }

521

522 if source_data and source_data.published then

523 metadata.creation_date = extract_full_date(source_data.published)

524 end

525

526 return metadata

527end

528-- }}}

529

530-- {{{ function extract_attachments

531local function extract_attachments(content_object)

532 -- Extract media attachment metadata from ActivityPub Note object

533 -- Returns nil if no attachments, or array of attachment metadata

534 if not content_object.attachment then

535 return nil

536 end

537

538 local attachments = {}

539 for _, attachment in ipairs(content_object.attachment) do

540 -- Only process Document type attachments (images, videos, etc.)

541 if attachment.type == "Document" and attachment.url then

542 -- Extract the relative path from the URL

543 -- URL format: https://server.com/media/files/123/456/789/original/filename.ext

544 -- We extract: files/123/456/789/original/filename.ext

545 local relative_path = attachment.url:match("/files/(.+)$")

546 if relative_path then

547 relative_path = "files/" .. relative_path

548 end

549

550 local attachment_entry = {

551 media_type = attachment.mediaType,

552 url = attachment.url,

553 relative_path = relative_path,

554 alt_text = attachment.name, -- May be nil if no alt text provided

555 width = attachment.width,

556 height = attachment.height,

557 blurhash = attachment.blurhash

558 }

559 table.insert(attachments, attachment_entry)

560 end

561 end

562

563 if #attachments > 0 then

564 return attachments

565 end

566 return nil

567end

568-- }}}

569

570local poems_json = {}

571local boost_count = 0

572local original_count = 0

573local attachment_count = 0

574local excluded_count = 0 -- Issue 6-031: Track excluded poems

575-- Issue 10-038: Separate ID numbering for fediverse_boost category

576-- Boosts get their own sequential IDs starting from 0001, independent of fediverse posts

577local boost_id_counter = 1

578

579print("🔄 Processing activities with privacy mode: " .. privacy_config.mode)

580print("🔄 Include boosts: " .. tostring(privacy_config.include_boosts))

581

582for key, activity in pairs(data.orderedItems) do

583 local activity_type, content_object = categorize_activity(activity)

584

585 -- Issue 6-031: Generate poem ID early for exclusion check

586 -- IDs are assigned before exclusion filter runs, preserving stable anchors

587 local poem_id = string.format("%04d", key)

588

589 if activity_type == "original_post" then

590 -- Issue 6-031: Check exclusion filter (tombstone - leaves gap in ID sequence)

591 if poem_exclusions:is_excluded("fediverse", poem_id) then

592 excluded_count = excluded_count + 1

593 goto continue

594 end

595

596 -- Process original posts (Create activities)

597 local cw = content_object.summary or ""

598 local content = content_object.content

599

600 -- Process content with privacy settings

601 local processed_content = process_fediverse_content(content, cw, privacy_config.mode)

602 if processed_content then

603 local poem_entry = {

604 id = poem_id,

605 category = "fediverse",

606 source_file = "outbox.json",

607 creation_date = extract_full_date(activity.published),

608 content_warning = processed_content.content_warning,

609 content = processed_content.content,

610 raw_content = processed_content.raw_content,

611 metadata = generate_poem_metadata(processed_content.content, cw, activity, processed_content.original_content)

612 }

613

614 -- Add privacy metadata

615 if processed_content.privacy_applied then

616 poem_entry.metadata.privacy_mode = privacy_config.mode

617 poem_entry.metadata.mentions_anonymized = true

618 if privacy_config.preserve_original_length then

619 poem_entry.metadata.original_character_count = string.len(processed_content.original_content)

620 end

621 end

622

623 -- Extract media attachments from the Note object

624 -- Attachments contain image/video URLs that map to local media_attachments directory

625 local attachments = extract_attachments(content_object)

626 if attachments then

627 poem_entry.attachments = attachments

628 poem_entry.metadata.has_attachments = true

629 poem_entry.metadata.attachment_count = #attachments

630 attachment_count = attachment_count + #attachments

631 end

632

633 table.insert(poems_json, poem_entry)

634 original_count = original_count + 1

635 end

636

637 elseif activity_type == "boost" and privacy_config.include_boosts then

638 -- Issue 10-038: Generate separate boost ID using boost_id_counter

639 -- Boosts have their own ID sequence: fediverse_boost/0001, 0002, etc.

640 local boost_id = string.format("%04d", boost_id_counter)

641

642 -- Issue 6-031: Check exclusion filter for boosts (using boost-specific ID)

643 if poem_exclusions:is_excluded("fediverse_boost", boost_id) then

644 excluded_count = excluded_count + 1

645 goto continue

646 end

647

648 -- Process boosted content when enabled

649 local boost_content = extract_boost_content(activity)

650 if boost_content then

651 -- Apply privacy processing to boost content too

652 local processed_boost = process_fediverse_content(boost_content.content, "", privacy_config.mode)

653 if processed_boost then

654 local boost_entry = {

655 id = boost_id,

656 category = "fediverse_boost",

657 source_file = "outbox.json",

658 creation_date = extract_full_date(activity.published),

659 content = processed_boost.content,

660 raw_content = processed_boost.raw_content,

661 metadata = boost_content.metadata

662 }

663

664 -- Add privacy metadata for boosts

665 if processed_boost.privacy_applied then

666 boost_entry.metadata.privacy_mode = privacy_config.mode

667 boost_entry.metadata.mentions_anonymized = true

668 end

669

670 table.insert(poems_json, boost_entry)

671 boost_count = boost_count + 1

672 -- Issue 10-038: Increment boost ID counter for next boost

673 boost_id_counter = boost_id_counter + 1

674 end

675 end

676 end

677

678 ::continue::

679end

680

681-- {{{ Generate JSON output for HTML generation

682-- Create output directory

683os.execute("mkdir -p " .. save_location)

684

685-- Count posts with attachments for statistics

686local posts_with_attachments = 0

687for _, poem in ipairs(poems_json) do

688 if poem.attachments then

689 posts_with_attachments = posts_with_attachments + 1

690 end

691end

692

693-- Generate JSON output

694local json_output = {

695 poems = poems_json,

696 extraction_summary = {

697 total_poems = #poems_json,

698 original_posts = original_count,

699 boosted_posts = boost_count,

700 poems_excluded = excluded_count, -- Issue 6-031: Excluded poem count

701 by_category = {

702 fediverse = original_count,

703 fediverse_boost = boost_count

704 },

705 content_warnings = {},

706 extraction_date = os.date("%Y-%m-%dT%H:%M:%SZ"),

707 privacy_settings = {

708 mode = privacy_config.mode,

709 include_boosts = privacy_config.include_boosts,

710 mentions_anonymized = (privacy_config.mode == "clean"),

711 anonymization_prefix = privacy_config.anonymization_prefix

712 },

713 attachment_statistics = {

714 total_attachments = attachment_count,

715 posts_with_attachments = posts_with_attachments

716 }

717 }

718}

719

720-- Collect unique content warnings

721local cw_set = {}

722for _, poem in ipairs(poems_json) do

723 if poem.content_warning then

724 cw_set[poem.content_warning] = true

725 end

726end

727for cw, _ in pairs(cw_set) do

728 table.insert(json_output.extraction_summary.content_warnings, cw)

729end

730

731local json_file = save_location .. "/poems.json"

732local f = io.open(json_file, "w")

733f:write(dkjson.encode(json_output, { indent = true }))

734f:close()

735

736-- Issue 7-006: Full-line coloring for success messages

737print(COLOR_GREEN .. "✅ Fediverse extraction complete" .. COLOR_RESET)

738print(" 📄 Generated: " .. relative_path(json_file))

739print(" 📊 Total posts processed: " .. #poems_json)

740print(" 📝 Original posts: " .. original_count)

741print(" 🔄 Boosted posts: " .. boost_count)

742if excluded_count > 0 then

743 print(" 🚫 Excluded posts: " .. excluded_count .. " (tombstoned)")

744end

745print(" 🖼️ Attachments found: " .. attachment_count .. " in " .. posts_with_attachments .. " posts")

746print(" 🚨 Content warnings: " .. #json_output.extraction_summary.content_warnings)

747print(" 🔒 Privacy mode: " .. privacy_config.mode)

748if privacy_config.mode == "clean" then

749 print(" 🎭 Mentions anonymized: " .. user_counter - 1 .. " users")

750end

751-- }}}

752

753