src/main.lua

6local function parse_dir_from_args(args)

7 if not args then return nil end

8 for i = 1, #args do

9 local a = args[i]

10 -- Skip flags (start with -)

11 if a and a:sub(1, 1) ~= "-" then

12 return a

13 end

14 -- Handle --dir=path or --dir path

15 if a == "--dir" and args[i + 1] then

16 return args[i + 1]

17 end

18 if a:match("^--dir=") then

19 return a:match("^--dir=(.+)")

20 end

21 end

22 return nil

25local function setup_dir_path(provided_dir)

26 if provided_dir then

27 return provided_dir

28 end

29 return "/mnt/mtwo/programming/ai-stuff/neocities-modernization"

88local tui_available, menu = pcall(require, "menu")

89local tui_load_error = not tui_available and menu or nil -- menu contains error message when pcall fails

90local tui = tui_available and require("tui") or nil

397function M.is_data_fresh()

398 -- Check if assets/poems.json exists and is newer than source files

399 local output_file = utils.asset_path("poems.json")

400 if not utils.file_exists(output_file) then

401 return false

402 end

404 -- Get output file modification time

405 local output_mtime = utils.get_file_mtime(output_file)

406 if not output_mtime then

407 return false

408 end

410 -- Check source JSON files

411 local source_files = {

412 DIR .. "/input/fediverse/files/poems.json",

413 DIR .. "/input/messages/files/poems.json",

414 DIR .. "/input/notes/files/poems.json",

415 DIR .. "/input/bluesky/files/poems.json"

416 }

418 for _, source_file in ipairs(source_files) do

419 if utils.file_exists(source_file) then

420 local source_mtime = utils.get_file_mtime(source_file)

421 if source_mtime and source_mtime > output_mtime then

422 return false -- Source is newer, need to regenerate

423 end

424 end

425 end

427 return true -- Output exists and is up to date

432function M.extract_poems(force)

433 -- Skip if data is fresh (unless forced)

434 if not force and M.is_data_fresh() then

435 utils.log_info("Poem data is up to date, skipping extraction")

436 return true

437 end

439 local output_file = utils.asset_path("poems.json")

441 -- Use auto-detection to handle both JSON extracts and compiled.txt

442 local include_boosts = resolve_include_boosts()

443 if not include_boosts then

444 utils.log_info("Boosts excluded for this extraction (CLI/config: include_boosts=false)")

445 end

446 local success, result = pcall(function()

447 return poem_extractor.extract_poems_auto(DIR, output_file, { include_boosts = include_boosts })

448 end)

450 if success then

451 local mode = result.metadata.source_mode

452 utils.log_info("Poem extraction completed using " .. mode .. " mode")

453 utils.log_info("Found " .. result.metadata.total_poems .. " poems")

454 return true

455 else

456 utils.log_error("Poem extraction failed: " .. tostring(result))

457 return false

458 end

463function M.validate_poems()

464 local input_file = utils.asset_path("poems.json")

465 local output_file = utils.asset_path("validation-report.json")

466

467 if not utils.file_exists(input_file) then

468 utils.log_error("Poems file not found. Run extraction first.")

469 return false

470 end

471

472 poem_validator.validate_poems(input_file, output_file)

473 utils.log_info("Poem validation completed")

474 return true

479function M.test_embedding_service()

480 utils.log_info("Testing inference server embedding service...")

482 local server_manager = require("embedding-server-manager")

483 if server_manager then

484 local endpoint = server_manager.ensure_ready()

485 if endpoint then

486 server_manager.test_embedding(endpoint, inference_config.get_selected_model())

487 return true

488 end

489 end

491 utils.log_error("Embedding service test failed")

492 return false

498function M.catalog_images(verbose)

500 local success, result = pcall(function()

501 return image_manager.main(verbose)

502 end)

504 if success and result then

505 utils.log_info("Image cataloging completed successfully")

506 return true

507 else

508 utils.log_error("Image cataloging failed: " .. tostring(result))

509 return false

510 end

515function M.is_html_fresh()

516 -- Check if output HTML files exist and are newer than source data

517 -- We check chronological.html as the main indicator

518 local output_file = DIR .. "/output/chronological.html"

519 if not utils.file_exists(output_file) then

520 return false

521 end

523 local output_mtime = utils.get_file_mtime(output_file)

524 if not output_mtime then

525 return false

526 end

528 -- Check against poems.json (the primary source data)

529 local poems_file = utils.asset_path("poems.json")

530 if utils.file_exists(poems_file) then

531 local poems_mtime = utils.get_file_mtime(poems_file)

532 if poems_mtime and poems_mtime > output_mtime then

533 return false

534 end

535 end

537 -- Check against similarity matrix (affects similarity/different pages)

538 local similarity_file = utils.embeddings_dir() .. "/similarity_matrix.json"

539 if utils.file_exists(similarity_file) then

540 local similarity_mtime = utils.get_file_mtime(similarity_file)

541 if similarity_mtime and similarity_mtime > output_mtime then

542 return false

543 end

544 end

546 return true

555function M.generate_website_html(force, pages_spec, poems_per_page, num_threads, chrono_per_page)

556 -- Skip if HTML is fresh (unless forced)

557 if not force and M.is_html_fresh() then

558 utils.log_info("Website HTML is up to date, skipping generation")

559 return true

560 end

562 -- Check dependencies

563 local poems_file = utils.asset_path("poems.json")

564 if not utils.file_exists(poems_file) then

565 utils.log_error("Poems file not found. Run extraction first.")

566 return false

567 end

569 -- Issue 10-033: Check that pre-computed caches exist (these are what actually get used)

570 -- The similarity_matrix.json and embeddings.json are NOT loaded anymore - saves 739MB RAM

571 -- Issue 10-054: diversity stays on disk (embeddings_dir_disk); the similarity

572 -- ranking cache is movable (embeddings_dir, RAM once the switch is flipped).

573 local diversity_cache_file = utils.embeddings_dir_disk() .. "/diversity_cache.json"

574 local similarity_cache_file = utils.embeddings_dir() .. "/similarity_rankings_cache.json"

576 -- Pre-flight: validate the prerequisite caches EXIST before the long

577 -- generation starts. A miss is a hard error (return false -> the CLI handler

578 -- os.exit(1)s -> run.sh stops). The path is printed because the usual cause is

579 -- a RAM/disk mismatch (Issue 10-054): a writer left it on disk while this

580 -- reader looks in RAM (or vice versa), so seeing WHERE it looked is the clue.

581 if not utils.file_exists(diversity_cache_file) then

582 utils.log_error("Diversity cache not found at: " .. diversity_cache_file

583 .. " (run ./run.sh --generate-diversity, or check the RAM/disk switch)")

584 return false

585 end

587 if not utils.file_exists(similarity_cache_file) then

588 utils.log_error("Similarity rankings cache not found at: " .. similarity_cache_file

589 .. " (run ./run.sh --generate-similarity, or check the RAM/disk switch)")

590 return false

591 end

593 -- Load only poems data (12MB) - caches are loaded inside generator as needed

594 -- Issue 10-033: Skip loading embeddings.json (77MB) and similarity_matrix.json (662MB)

595 -- These are never used - generator functions use pre-computed caches exclusively

596 local poems_data = utils.read_json_file(poems_file)

597 if not poems_data then

598 utils.log_error("Failed to load poems data")

599 return false

600 end

602 -- Note: similarity_data and embeddings_data parameters below are nil

603 -- This is safe because generator functions use cache lookups (Issue 10-033)

604 local similarity_data = nil

605 local embeddings_data = nil

607 local output_dir = DIR .. "/output"

609 -- Generate chronological index (main entry point)

610 -- Issue 9-003: Pass chrono_per_page for CLI override of chronological pagination

611 local success = flat_html_generator.generate_chronological_index_with_navigation(poems_data, output_dir, chrono_per_page)

612 if not success then

613 utils.log_error("Failed to generate chronological index")

614 return false

615 end

617 -- Generate explore.html (the map) + explore-2.html (the deeper math).

618 -- Pass poems_data so both pages render live corpus stats instead of prose.

619 flat_html_generator.generate_simple_discovery_instructions(output_dir, poems_data)

621 -- Generate all similarity and diversity pages

622 -- Note: This is the long operation - generates ~12,000+ files

623 -- Phase D (Issue 8-012): Pass pages_spec for pagination control

624 -- Issue 8-022: Pass poems_per_page for CLI override

625 -- Issue 9-002: Pass num_threads for parallel processing

626 -- Issue 9-003: Pass chrono_per_page for chronological mapping in parallel workers

627 local gen_success = flat_html_generator.generate_complete_flat_html_collection(

628 poems_data, similarity_data, embeddings_data, output_dir, pages_spec, poems_per_page, num_threads, chrono_per_page

629 )

631 if gen_success then

632 utils.log_info("Website HTML generation completed successfully")

633 return true

634 else

635 utils.log_error("Website HTML generation failed")

636 return false

637 end

642function M.generate_complete_dataset()

643 utils.log_info("Generating complete dataset...")

645 if not M.extract_poems() then

646 return false

647 end

649 if not M.validate_poems() then

650 return false

651 end

653 if not M.catalog_images() then

654 return false

655 end

657 utils.log_info("Complete dataset generation finished")

658 return true

663function M.show_project_status()

664 print("\n=== PROJECT STATUS ===")

666 local paths = utils.get_project_paths(DIR)

667 local assets_root = utils.get_assets_root()

669 -- Check key files (input files use paths.root, generated assets use assets_root)

670 local status_items = {

671 {"Legacy Source", paths.root .. "/compiled.txt"},

672 {"JSON Extracts (Fed)", paths.root .. "/input/fediverse/files/poems.json"},

673 {"JSON Extracts (Msg)", paths.root .. "/input/messages/files/poems.json"},

674 {"JSON Extracts (Notes)", paths.root .. "/input/notes/files/poems.json"},

675 {"JSON Extracts (Sky)", paths.root .. "/input/bluesky/files/poems.json"},

676 {"Processed Poems", utils.asset_path("poems.json")},

677 {"Validation Report", utils.asset_path("validation-report.json")},

678 {"Image Catalog", utils.asset_path("image-catalog.json")},

679 {"Poem Extractor", paths.src .. "/poem-extractor.lua"},

680 {"Poem Validator", paths.src .. "/poem-validator.lua"},

681 {"Image Manager", paths.src .. "/image-manager.lua"},

682 {"Embedding Server Manager", paths.src .. "/embedding-server-manager.lua"}

683 }

685 -- Show assets location

686 print(string.format("%-20s: %s", "Assets Location", assets_root))

688 for _, item in ipairs(status_items) do

689 local name, filepath = item[1], item[2]

690 local status = utils.file_exists(filepath) and "✅ Found" or "❌ Missing"

691 print(string.format("%-20s: %s", name, status))

692 end

694 -- Show poem count if available

695 local poems_file = utils.asset_path("poems.json")

696 if utils.file_exists(poems_file) then

697 local content = utils.read_file(poems_file)

698 if content then

699 local poem_count = select(2, content:gsub('"id":', '"id":'))

700 print(string.format("%-20s: %d poems", "Dataset Size", poem_count))

701 end

702 end

704 print("")

709function M.clean_and_rebuild()

710 utils.log_info("Cleaning and rebuilding assets...")

712 if utils.confirm_action("This will delete existing assets. Continue?") then

713 -- Remove old generated assets (from configured assets location)

714 os.execute("rm -f " .. utils.asset_path("poems.json"))

715 os.execute("rm -f " .. utils.asset_path("validation-report.json"))

717 -- Regenerate

718 return M.generate_complete_dataset()

719 else

720 utils.log_info("Operation cancelled")

721 return false

722 end

729function M.handle_tui_action(values)

730 local executed = false

732 -- Data Pipeline actions

733 if values.extract == "1" then

734 M.extract_poems(true) -- force=true in interactive mode

735 executed = true

736 end

737 if values.validate == "1" then

738 M.validate_poems()

739 executed = true

740 end

741 if values.catalog == "1" then

742 M.catalog_images()

743 executed = true

744 end

745 if values.dataset == "1" then

746 M.generate_complete_dataset()

747 executed = true

748 end

750 -- Embedding & Similarity actions

751 if values.test_inference_server == "1" then

752 M.test_embedding_service()

753 executed = true

754 end

755 if values.similarity == "1" then

756 -- Run similarity on the GPU. The CPU engine was removed (Issue 10-057) -- these

757 -- are O(N^2) calculations that make no sense on a CPU. vk_similarity uses

758 -- absolute/project-root-relative paths, so it runs fine in-process here (no

759 -- cd-wrapper needed, unlike diversity before the path unification).

760 utils.log_info("Running GPU similarity calculation...")

761 -- vk_similarity reads DIR from the environment to locate its library; set it

762 -- before the module loads.

763 local ffi = require("ffi")

764 ffi.cdef[[int setenv(const char *name, const char *value, int overwrite);]]

765 ffi.C.setenv("DIR", DIR, 1)

766 local vk_sim = require("vulkan-compute.lua.vk_similarity")

767 -- Same K the pipeline uses: pages x poems_per_page from config (the default

768 -- minimum_pages here, since this action takes no --pages). A capped cache the

769 -- HTML stage cannot fill is caught by the loader's K-check.

770 local pag = config_loader.load().pagination

771 if not pag then error("config.pagination missing; cannot size the similarity cache") end

772 local top_k = pag.minimum_pages * pag.poems_per_page

773 local thread_count = tonumber(values.thread_count) or 8

774 local embeddings_file = utils.embeddings_dir() .. "/embeddings.json"

775 vk_sim.generate_similarity_matrix_gpu_parallel(

776 embeddings_file, inference_config.get_selected_model(), false, thread_count, top_k)

777 executed = true

778 end

780 -- HTML Generation actions

781 if values.chronological == "1" then

782 local poems_file = utils.asset_path("poems.json")

783 local poems_data = utils.read_json_file(poems_file)

784 if poems_data then

785 flat_html_generator.generate_chronological_index_with_navigation(poems_data, DIR .. "/output")

786 utils.log_info("Generated chronological.html")

787 end

788 executed = true

789 end

790 if values.explore == "1" then

791 flat_html_generator.generate_simple_discovery_instructions(DIR .. "/output")

792 utils.log_info("Generated explore.html")

793 executed = true

794 end

795 if values.similar_pages == "1" then

796 utils.log_info("Generating similarity pages (use scripts/generate-html-parallel --similar-only)...")

797 os.execute("luajit " .. DIR .. "/scripts/generate-html-parallel --similar-only")

798 executed = true

799 end

800 if values.different_pages == "1" then

801 utils.log_info("Generating difference pages (use scripts/generate-html-parallel --different-only)...")

802 os.execute("luajit " .. DIR .. "/scripts/generate-html-parallel --different-only")

803 executed = true

804 end

805 if values.full_website == "1" then

806 M.generate_website_html(true, nil, nil) -- TUI mode: use config defaults

807 executed = true

808 end

810 -- Testing actions

811 if values.test_similar == "1" then

812 local poem_id = tonumber(values.test_poem_id) or 1

813 M.test_single_similarity_page(poem_id)

814 executed = true

815 end

816 if values.test_different == "1" then

817 local poem_id = tonumber(values.test_poem_id) or 1

818 M.test_single_difference_page(poem_id)

819 executed = true

820 end

822 -- Utility actions

823 if values.status == "1" then

824 M.show_project_status()

825 executed = true

826 end

827 if values.clean == "1" then

828 M.clean_and_rebuild()

829 executed = true

830 end

832 return executed

838function M.test_single_similarity_page(poem_id)

839 utils.log_info("Testing similarity page for poem " .. poem_id .. "...")

840 local poems_file = utils.asset_path("poems.json")

841 local similarity_file = utils.embeddings_dir() .. "/similarity_matrix.json"

842 local output_dir = DIR .. "/output"

844 local poems_data = utils.read_json_file(poems_file)

845 local similarity_data = utils.read_json_file(similarity_file)

847 if poems_data and similarity_data then

848 local poem_data = nil

849 for _, poem in ipairs(poems_data.poems) do

850 if poem.id == poem_id then

851 poem_data = poem

852 break

853 end

854 end

856 if poem_data then

857 local ranking = flat_html_generator.generate_similarity_ranked_list(poem_id, poems_data, similarity_data.similarities or similarity_data)

858 -- Issue 10-036: Pass nil for chrono_mapping in test function (uses "01" fallback)

859 local html = flat_html_generator.generate_flat_poem_list_html(poem_data, ranking, "similar", poem_id, nil)

860 local test_file = string.format("%s/test_similar_%03d.html", output_dir, poem_id)

861 os.execute("mkdir -p " .. output_dir)

862 utils.write_file(test_file, html)

863 utils.log_info("Test file written: " .. test_file)

864 else

865 utils.log_error("Poem ID " .. poem_id .. " not found")

866 end

867 else

868 utils.log_error("Failed to load required data files")

869 end

875function M.test_single_difference_page(poem_id)

876 utils.log_info("Testing difference page for poem " .. poem_id .. "...")

877 local poems_file = utils.asset_path("poems.json")

878 local embeddings_file = utils.embeddings_dir() .. "/embeddings.json"

879 local output_dir = DIR .. "/output"

881 local poems_data = utils.read_json_file(poems_file)

882 local embeddings_data = utils.read_json_file(embeddings_file)

884 if poems_data and embeddings_data then

885 local poem_data = nil

886 for _, poem in ipairs(poems_data.poems) do

887 if poem.id == poem_id then

888 poem_data = poem

889 break

890 end

891 end

893 if poem_data then

894 local diversity_chaining = require("diversity-chaining")

895 local ranking = diversity_chaining.generate_diversity_chain(poem_id, poems_data, embeddings_data)

896 -- Issue 10-036: Pass nil for chrono_mapping in test function (uses "01" fallback)

897 local html = flat_html_generator.generate_flat_poem_list_html(poem_data, ranking, "different", poem_id, nil)

898 local test_file = string.format("%s/test_different_%03d.html", output_dir, poem_id)

899 os.execute("mkdir -p " .. output_dir)

900 utils.write_file(test_file, html)

901 utils.log_info("Test file written: " .. test_file)

902 else

903 utils.log_error("Poem ID " .. poem_id .. " not found")

904 end

905 else

906 utils.log_error("Failed to load required data files")

907 end

914function M.main(options)

915 options = options or {}

917 if options.interactive then

918 -- Use TUI menu if available, otherwise fall back to simple menu

919 while true do

920 local action, values = M.show_tui_menu()

922 if action == "quit" then

923 print("Goodbye!")

924 break

925 elseif action == "run" then

926 local executed = M.handle_tui_action(values)

928 if executed then

929 -- Don't pause after status display

930 if values.status ~= "1" then

931 print("\nPress Enter to continue...")

932 io.read()

933 end

934 else

935 print("No action selected. Use space to toggle options, then press Enter or select [Run].")

936 print("\nPress Enter to continue...")

937 io.read()

938 end

939 end

940 end

941 elseif options.parse_only then

942 M.extract_poems(options.force)

943 elseif options.validate_only then

944 M.validate_poems()

945 elseif options.catalog_only then

946 -- Issue 10-015a: Pass verbose option for detailed statistics

947 M.catalog_images(options.verbose)

948 elseif options.html_only then

949 -- Run only HTML generation

950 -- Phase D (Issue 8-012): Pass pages parameter

951 -- Issue 8-022: Pass poems_per_page parameter

952 -- Issue 9-002: Pass threads parameter for parallel processing

953 -- Issue 9-003: Pass chrono_per_page parameter

954 -- A false return means a prerequisite was missing (e.g. the similarity

955 -- rankings cache) or generation failed. This MUST stop the pipeline with a

956 -- non-zero exit so run.sh's `|| exit 1` fires -- otherwise the run carries

957 -- on and builds a site whose nav pages were never made (Issue 10-054

958 -- fallout). A missing prerequisite is an error, never a warning to ignore.

959 if not M.generate_website_html(options.force, options.pages, options.poems_per_page, options.threads, options.chrono_per_page) then

960 os.exit(1)

961 end

962 else

963 -- Non-interactive mode - generate dataset and website HTML (full pipeline)

964 -- Phase D (Issue 8-012): Pass pages parameter

965 -- Issue 8-022: Pass poems_per_page parameter

966 -- Issue 9-002: Pass threads parameter for parallel processing

967 -- Issue 9-003: Pass chrono_per_page parameter

968 utils.log_info("Running in non-interactive mode (full pipeline)")

969 M.show_project_status()

970 M.generate_complete_dataset()

971 -- Same as above: a false return is a hard failure that must stop the run.

972 if not M.generate_website_html(options.force, options.pages, options.poems_per_page, options.threads, options.chrono_per_page) then

973 os.exit(1)

974 end

975 end