scripts/deploy-to-neocities

#!/usr/bin/env luajit
-- {{{ deploy-to-neocities
-- Chunked, adaptive, resumable deploy of output/ to the live site's
-- similar-different/. Replaces the stock neocities push path that uploads one
-- file per request, never backs off, and dies partway under rate limiting.
--
-- General description (for a CEO): the old deploy fired tens of thousands of
-- single-file requests at the server until it got tired and hung up, losing the
-- run. This sends many files per request, speeds up while the server is happy and
-- backs off the instant it pushes back, and -- because it first asks the live
-- site what it already has -- it only sends what is missing, so a re-run resumes
-- where the last one stopped instead of starting over.
--
-- Pieces: libs/neocities-api.lua (status-honest curl layer), libs/neocities-sync
-- (the tested adaptive control loop + diff). This script wires them to the real
-- build and the real key.
--
-- Usage:
-- scripts/deploy-to-neocities [DIR] [--dry-run] [--yes] [--prune] [--only PREFIX]
-- --dry-run list + diff only; change nothing
-- --yes skip the confirmation prompts
-- --prune after uploading, DELETE remote files under similar-different/
-- that are not in the local build (a true mirror). Destructive;
-- asks separately. Upload-then-prune keeps the site working
-- throughout (current content goes up before stale is removed).
-- The COMPLETE stale list is written to tmp/prune-list (RAM),
-- whether or not you confirm the delete; the terminal shows
-- the first 50 and points at the file for the rest.
-- --only PREFIX restrict BOTH passes to remote paths under PREFIX (e.g.
-- similar-different/gallery) -- for testing on a small slice
-- }}}

local DIR = "/mnt/mtwo/programming/ai-stuff/neocities-modernization"
local dry_run, assume_yes, only_prefix, prune = false, false, nil, false

-- {{{ parse args
do
local i = 1
while i <= #arg do
local a = arg[i]
if a == "--dry-run" then dry_run = true
elseif a == "--yes" then assume_yes = true
elseif a == "--prune" then prune = true
elseif a == "--only" then only_prefix = arg[i + 1]; i = i + 1
elseif a ~= "" and a:sub(1, 2) ~= "--" then DIR = a end
i = i + 1
end
end
package.path = DIR .. "/libs/?.lua;" .. DIR .. "/?.lua;" .. package.path
local dkjson = require("dkjson")
local api = require("neocities-api")
local sync = require("neocities-sync")
local socket = require("socket") -- for socket.sleep (politeness/backoff)
local REMOTE_DIR = "similar-different"
-- }}}

-- {{{ load_key()
local function load_key()
local cfg = (os.getenv("HOME") or "") .. "/.config/neocities/config.json"
local f = io.open(cfg, "r"); if not f then error("no API key file: " .. cfg) end
local data = dkjson.decode(f:read("*a")); f:close()
local key = data and data.API_KEY
if not key or key == "" then error("no API_KEY in " .. cfg) end
return key
end
-- }}}

-- {{{ local_files() -- enumerate output/ with sizes AND sha1, applying the
-- deploy's excludes (debug-logs/, the GPU bin, anchored at output/ root). Returns
-- array of { remote=, abspath=, cost=bytes, bytes=, sha1= }. sha1 lets us upload
-- only files whose CONTENT differs from the live copy (so a changed build updates
-- stale pages, not just missing ones). bytes/cost are the same value (diff_upload
-- sorts by .bytes; the adaptive loop batches by .cost).
local function local_files()
local size_of = {}
local h = io.popen(string.format("find %q/output -type f -printf '%%s\\t%%P\\n' 2>/dev/null", DIR))
for line in h:lines() do
local s, rel = line:match("^(%d+)\t(.+)$")
if rel then size_of[rel] = tonumber(s) end
end
h:close()

local items = {}
-- one xargs sha1sum pass over relative paths (reads the build once to hash it)
local g = io.popen(string.format(
"cd %q/output && find . -type f -print0 | xargs -0 sha1sum 2>/dev/null", DIR))
for line in g:lines() do
local sha, rel = line:match("^(%x+)%s+%./(.+)$")
if rel and not rel:match("^debug%-logs/") and rel ~= "diversity-cache-gpu-batch.bin" then
local sz = size_of[rel] or 0
items[#items + 1] = {
remote = REMOTE_DIR .. "/" .. rel,
abspath = DIR .. "/output/" .. rel,
cost = sz, bytes = sz, sha1 = sha,
}
end
end
g:close()
return items
end
-- }}}

-- {{{ remote_sha() -- { remote_path -> sha1_hash } for the live similar-different/
-- Uses the no-path (full, RECURSIVE) site listing -- a path-scoped list returns
-- only one directory level, which would make the diff think everything nested is
-- missing. Filter to our section by prefix.
local function remote_sha()
local files, err = api.list(nil)
if not files then error("could not list the live site: " .. tostring(err)) end
local by_path, prefix = {}, REMOTE_DIR .. "/"
for _, e in ipairs(files) do
if not e.is_directory and e.path and e.path:sub(1, #prefix) == prefix then
by_path[e.path] = e.sha1_hash
end
end
return by_path
end
-- }}}

-- {{{ main
local key = load_key()
api.set_key(key)
api.set_project_root(DIR)

-- {{{ confirm(prompt) -- y/N gate, auto-yes under --yes
local function confirm(prompt)
if assume_yes then return true end
io.write(prompt .. " [y/N] ")
local ans = io.read("*l")
return ans == "y" or ans == "Y"
end
-- }}}

io.write("Hashing the local build and listing the live site ...\n")
local remote = remote_sha()
local locals = local_files()

-- content diff (upload set): files absent OR whose sha1 differs from the live copy.
local need = sync.diff_upload(locals, remote)
local todo, todo_bytes = {}, 0
for _, it in ipairs(need) do
if not only_prefix or it.remote:sub(1, #only_prefix) == only_prefix then
todo[#todo + 1] = it; todo_bytes = todo_bytes + (it.cost or 0)
end
end

-- prune set: remote files (under similar-different/) not present locally.
local local_set = {}
for _, it in ipairs(locals) do local_set[it.remote] = true end
local stale = {}
if prune then
local remote_entries = {}
for p in pairs(remote) do remote_entries[#remote_entries + 1] = { path = p, is_directory = false } end
for _, p in ipairs(sync.diff_delete(remote_entries, local_set)) do
if not only_prefix or p:sub(1, #only_prefix) == only_prefix then stale[#stale + 1] = p end
end
end

local present_n = 0; for _ in pairs(remote) do present_n = present_n + 1 end
io.write(string.format("Local files: %d On site now: %d Upload (new/changed): %d (%.1f GB)%s%s\n",
#locals, present_n, #todo, todo_bytes / 1e9,
prune and string.format(" Prune (stale on remote): %d", #stale) or "",
only_prefix and (" [--only " .. only_prefix .. "]") or ""))

if dry_run then io.write("[--dry-run] Stopping before any change.\n"); return end

-- {{{ bucket_of(remote) -- top-level dir under similar-different/ (or "other")
-- Plain string strip, NOT a Lua pattern: REMOTE_DIR contains "-", which is a
-- magic pattern character (lazy quantifier), so a pattern would fail to match.
local function bucket_of(remote)
local prefix = REMOTE_DIR .. "/"
local rel = (remote:sub(1, #prefix) == prefix) and remote:sub(#prefix + 1) or remote
return rel:match("^([^/]+)/") or "other"
end
-- }}}

-- {{{ UPLOAD pass
if #todo > 0 then
if not confirm(string.format("Upload %d files to the LIVE %s/ now?", #todo, REMOTE_DIR)) then
io.write("Aborted; nothing uploaded.\n"); return
end

-- Shuffle so batches are a random mix of directories -- every per-directory
-- bar then advances together instead of one finishing before the next starts.
-- Order does not affect the result, so this is purely for the nicer view; it
-- only slightly delays when the adaptive loop first meets a big batch.
math.randomseed(os.time())
for k = #todo, 2, -1 do local j = math.random(k); todo[k], todo[j] = todo[j], todo[k] end

local opts = { budget = 8 * 1024 * 1024, max_count = 60, base_delay = 0.2, sleep = socket.sleep }
local use_bars = (require("progress-display").mode() == 1) -- animated only on a TTY

if use_bars then
-- one bar per directory; redraw the whole block in place each frame
local order, total, done, status = {}, {}, {}, ""
for _, it in ipairs(todo) do
local b = bucket_of(it.remote)
if not total[b] then order[#order + 1] = b; total[b] = 0 end
total[b] = total[b] + 1
end
table.sort(order)
for _, b in ipairs(order) do done[b] = 0 end

-- Name-column width = the longest directory name present, so the bar +
-- counts columns line up no matter how long a name is (e.g.
-- "model-evaluation" no longer cuts into the bar). Names are ASCII, so
-- %-<width>s pads by the right number of display columns.
local BAR_W, first, namew = 28, true, 1
for _, b in ipairs(order) do if #b > namew then namew = #b end end
local bar_fmt = "\27[2K %-" .. namew .. "s %s %6d/%-6d %3d%%\n"
local function render()
if not first then io.write(string.format("\27[%dA", #order + 1)) end
first = false
local td, tt = 0, 0
for _, b in ipairs(order) do td = td + done[b]; tt = tt + total[b] end
io.write("\27[2K", string.format("Uploading to %s/ %d/%d %s\n", REMOTE_DIR, td, tt, status))
for _, b in ipairs(order) do
local t, d = total[b], done[b]
local frac = t > 0 and d / t or 0; if frac > 1 then frac = 1 end
local filled = math.floor(frac * BAR_W + 0.5)
io.write(string.format(bar_fmt, b,
string.rep("\226\150\136", filled) .. string.rep("\226\150\145", BAR_W - filled),
d, t, math.floor(frac * 100 + 0.5)))
end
io.flush()
end
render()
opts.log = function(m) status = m; render() end -- adaptive state in the header
opts.on_batch_ok = function(batch)
for _, it in ipairs(batch) do
local b = bucket_of(it.remote); done[b] = (done[b] or 0) + 1
end
render()
end
else
opts.log = function(m) io.write(" [adapt] " .. m .. "\n") end
end

local stats, err = sync.run_adaptive(todo, api.upload_batch, opts)
io.write("\n")
if not stats then
io.write("DEPLOY STOPPED: " .. tostring(err) .. "\nRe-run to resume (uploaded files are skipped).\n")
os.exit(1)
end
io.write(string.format("Uploaded %d files in %d requests (%d shrinks, %d throttles).\n",
stats.done, stats.requests, stats.shrinks, stats.throttled))
else
io.write("Nothing to upload -- content already in sync.\n")
end
-- }}}

-- {{{ write_prune_list(paths) -- persist the FULL stale set, one path per line,
-- to the RAM-backed tmp/ so the complete list outlives the terminal scrollback
-- and can be fed to other tools (sort, grep, a manual delete) -- even when it is
-- far longer than is sane to eyeball. Returns the absolute path written.
--
-- We deliberately do NOT shell out to ensure-tmp-symlink here: the listing step
-- above already drove a curl through tmp/neocities-curl.cfg, so by the time a
-- prune preview runs the tmp/ tmpfs symlink is provably materialised. (It also
-- honours the project rule against using exec for directory-targeting work.) If
-- tmp/ somehow is not there, the assert fails loudly rather than papering over a
-- missing directory -- a real error we would want surfaced, not swallowed.
local function write_prune_list(paths)
local path = DIR .. "/tmp/shared-memory/prune-list"
local f = assert(io.open(path, "w"), "could not write prune list: " .. path)
for _, p in ipairs(paths) do f:write(p, "\n") end
f:close()
return path
end
-- }}}

-- {{{ PRUNE pass (destructive; separate confirmation + a preview)
if prune then
if #stale == 0 then
io.write("Nothing to prune -- no remote files are absent locally.\n")
else
-- Write the complete list to RAM first; the file is the durable record
-- (the WHOLE stale set, every run). The terminal then shows only the
-- first PREVIEW_N so the y/N prompt below stays on screen even when the
-- stale set is thousands of paths -- anything past the cap lives only in
-- the file, which the trailing line points at.
local PREVIEW_N = 50
local list_path = write_prune_list(stale)
io.write(string.format("\n%d remote file(s) under %s/ are NOT in the local build", #stale, REMOTE_DIR))
if present_n > 0 then io.write(string.format(" (%.0f%% of what is on the site)", 100 * #stale / present_n)) end
io.write(":\n")
local shown = math.min(PREVIEW_N, #stale)
for i = 1, shown do io.write(" " .. stale[i] .. "\n") end
if #stale > shown then
io.write(string.format(" ... and %d more -- full list in %s\n", #stale - shown, list_path))
else
io.write(string.format("(full list also written to %s)\n", list_path))
end
if not confirm(string.format("DELETE these %d file(s) from the LIVE site?", #stale)) then
io.write("Skipped prune; uploads (if any) are live.\n"); return
end
local del_items = {}
for _, p in ipairs(stale) do del_items[#del_items + 1] = { path = p, cost = 1 } end
local dstats, derr = sync.run_adaptive(del_items, function(batch)
local paths = {}
for _, it in ipairs(batch) do paths[#paths + 1] = it.path end
return api.delete_batch(paths)
end, {
budget = 50, max_count = 50, base_delay = 0.2, -- count-based; deletes per request
sleep = socket.sleep, log = function(m) io.write(" [adapt] " .. m .. "\n") end,
})
io.write("\n")
if not dstats then
io.write("PRUNE STOPPED: " .. tostring(derr) .. "\nRe-run --prune to continue (already-deleted files are gone).\n")
os.exit(1)
end
io.write(string.format("Pruned %d files in %d requests (%d shrinks, %d throttles).\n",
dstats.done, dstats.requests, dstats.shrinks, dstats.throttled))
end
end
-- }}}
-- }}}