libs/neocities-sync.lua

1-- {{{ neocities-sync.lua

2-- The brains of the chunked, adaptive Neocities deploy: what to change, and how

3-- big a bite to take. Pure logic with NO network and NO IO of its own -- the

4-- actual HTTP calls and the clock are injected -- so the fiddly parts (the diff,

5-- the batching, and the size-adaptation control loop) are unit-testable offline

6-- against a mock API. The thin curl layer that really talks to neocities.org

7-- lives in scripts/neocities-sync; this file decides what that layer is told.

8--

9-- General description (for a CEO): the old deploy tried to delete a whole folder

10-- and upload thousands of files in single requests; the server timed out and the

11-- run died. This module instead computes the SMALLEST set of changes (only files

12-- that are stale or differ), groups them into right-sized batches, and -- like a

13-- driver feeling for a speed limit -- speeds up while requests succeed and backs

14-- off hard the moment the server pushes back, settling at a rate that holds.

15--

16-- Why this shape (lessons paid for): a single giant request times out (HTML error

17-- page, not JSON); deleting a directory makes the server do unbounded work; and

18-- treating "too big" and "too many requests" the same makes rate-limiting worse

19-- (smaller batches => MORE requests). So the control loop distinguishes them.

20-- }}}

24-- {{{ function M.diff_delete(remote_paths, local_set)

25-- Stale remote files = present on the live site but NOT in the local build. These

26-- are all that a mirror needs to remove -- usually a handful, never the whole

27-- tree -- so we never ask the server to delete a directory. Directories are

28-- skipped (neocities prunes empty ones; deleting a dir path is the very thing

29-- that timed out). remote_paths: array of { path=, is_directory= }. local_set:

30-- set { [path]=true } of files the build produced. Returns an array of paths.

31function M.diff_delete(remote_paths, local_set)

32 local stale = {}

33 for _, entry in ipairs(remote_paths) do

34 if not entry.is_directory and not local_set[entry.path] then

35 stale[#stale + 1] = entry.path

36 end

37 end

38 table.sort(stale)

39 return stale

40end

41-- }}}

43-- {{{ function M.diff_upload(local_items, remote_by_path)

44-- Files that need uploading = those whose content hash differs from the live copy

45-- (or that the live site lacks). This is what makes a re-run resumable and cheap:

46-- a second pass after a failure re-uploads only what did not land. local_items:

47-- array of { remote=, abspath=, bytes=, sha1= }. remote_by_path: { [remote]=sha1 }.

48-- Returns the subset of local_items still needing upload (largest first, so a too-

49-- big batch is discovered early when the control loop is still calibrating).

50function M.diff_upload(local_items, remote_by_path)

51 local need = {}

52 for _, item in ipairs(local_items) do

53 if remote_by_path[item.remote] ~= item.sha1 then

54 need[#need + 1] = item

55 end

56 end

57 table.sort(need, function(a, b) return (a.bytes or 0) > (b.bytes or 0) end)

58 return need

59end

60-- }}}

62-- {{{ function M.take_batch(items, start, budget, max_count)

63-- Greedily take items[start..] into one batch bounded by BOTH a cost budget and a

64-- count cap. cost is item.cost (bytes for uploads; 1 for deletes -- so deletes are

65-- count-limited and uploads are byte-limited, the constraint each operation

66-- actually has). Always takes at least one item, even if it alone exceeds the

67-- budget, so a single oversized file can still be attempted (and fail loudly)

68-- rather than wedging the loop. Returns (batch_array, next_start).

69function M.take_batch(items, start, budget, max_count)

70 local batch = {}

71 local sum = 0

72 local i = start

73 while i <= #items do

74 local cost = items[i].cost or 1

75 -- stop if adding this item would exceed a budget we have already filled

76 if #batch > 0 and (sum + cost > budget or #batch >= max_count) then

77 break

78 end

79 batch[#batch + 1] = items[i]

80 sum = sum + cost

81 i = i + 1

82 if #batch >= max_count then break end

83 end

84 return batch, i

85end

86-- }}}

88-- {{{ classify(status)

89-- Map an HTTP status to the control loop's reaction. Kept in one table so the

90-- policy is readable and testable in isolation:

91-- ok -> success; consider speeding up

92-- throttle -> 429 (rate limited): WAIT, do not shrink (shrinking => more

93-- requests => worse). Honor Retry-After.

94-- too_big -> 413 / 5xx / a non-JSON body on a 200: the request was too large

95-- or the server-side op timed out -> halve the batch and retry.

96-- fatal -> 400/401/403/404 and friends: a real error (auth, bad request);

97-- stop, because retrying or resizing cannot fix it.

98local function classify(status, ok_body)

99 if status == 200 and ok_body then return "ok" end

100 if status == 429 then return "throttle" end

101 if status == 408 or status == 413 or status == 0 -- 0 = curl transport error

102 or (status >= 500 and status <= 599)

103 or (status == 200 and not ok_body) then -- 200 but HTML/garbage body

104 return "too_big"

105 end

106 return "fatal"

107end

108M._classify = classify

109-- }}}

111-- {{{ function M.run_adaptive(items, op, opts)

112-- The AIMD control loop. Walks `items` front-to-back, forming a batch sized to the

113-- current budget, handing it to op(batch) -> { status=, ok_body=, retry_after= },

114-- and adjusting:

115-- too_big -> budget = max(floor, budget/2); retry the SAME items smaller.

116-- throttle -> sleep(retry_after or growing backoff); retry; nudge the inter-

117-- request delay up so we stop tripping it; budget unchanged.

118-- ok -> advance; after `grow_after` consecutive oks, budget *= grow (a

119-- gentle additive-ish increase, capped) -- AIMD: ease up, slam down.

120-- fatal -> stop and report.

121-- opts (all optional): budget, max_count, floor, grow, grow_after, max_budget,

122-- base_delay, max_retries, sleep(fn), log(fn). Returns a stats table including the

123-- converged budget so the caller can persist it for next time.

124function M.run_adaptive(items, op, opts)

125 opts = opts or {}

126 local budget = opts.budget or 8 * 1024 * 1024 -- 8 MB / batch to start

127 local max_count = opts.max_count or 200 -- and never > 200 files

128 local floor = opts.floor or 1

129 local grow = opts.grow or 1.5

130 local grow_after = opts.grow_after or 3

131 local max_budget = opts.max_budget or 64 * 1024 * 1024

132 local base_delay = opts.base_delay or 0 -- politeness between calls

133 local max_retries = opts.max_retries or 8

134 local sleep = opts.sleep or function() end

135 local log = opts.log or function() end

136 local on_batch_ok = opts.on_batch_ok or function() end -- called(batch) after each accepted batch

137

138 local stats = { requests = 0, ok = 0, throttled = 0, shrinks = 0, grows = 0,

139 done = 0, total = #items }

140 local i = 1

141 local streak = 0

142 local delay = base_delay

143

144 while i <= #items do

145 local batch, nexti = M.take_batch(items, i, budget, max_count)

146 local result, kind

147 local attempt = 0

148 repeat

149 attempt = attempt + 1

150 stats.requests = stats.requests + 1

151 result = op(batch) or { status = 0 }

152 kind = classify(result.status, result.ok_body)

153

154 if kind == "throttle" then

155 stats.throttled = stats.throttled + 1

156 local wait = result.retry_after or math.min(60, 2 ^ attempt)

157 delay = math.max(delay, 1) -- become permanently politer

158 log(string.format("throttled (429); waiting %ss [batch=%d]", wait, #batch))

159 sleep(wait)

160 elseif kind == "too_big" then

161 stats.shrinks = stats.shrinks + 1

162 streak = 0

163 if #batch <= floor then

164 -- already at the smallest unit and still failing: not a size

165 -- problem we can chunk our way out of -- surface it.

166 return nil, string.format(

167 "request failed at minimum batch size (status %s) on %s",

168 tostring(result.status), tostring(batch[1] and (batch[1].remote

169 or batch[1]) or "?")), stats

170 end

171 budget = math.max(floor, math.floor(budget / 2))

172 log(string.format("too big (status %s); halving budget -> %d",

173 tostring(result.status), budget))

174 batch, nexti = M.take_batch(items, i, budget, max_count) -- re-form smaller

175 end

176 until kind == "ok" or kind == "fatal" or attempt >= max_retries

177

178 if kind == "fatal" then

179 return nil, string.format("fatal API error (status %s)", tostring(result.status)), stats

180 end

181 if kind ~= "ok" then

182 return nil, string.format("gave up after %d attempts (status %s)",

183 attempt, tostring(result.status)), stats

184 end

185

186 -- success: advance past the batch

187 stats.ok = stats.ok + 1

188 stats.done = stats.done + #batch

189 on_batch_ok(batch) -- let callers update per-item progress displays

190 i = nexti

191 streak = streak + 1

192 if streak >= grow_after and budget < max_budget then

193 budget = math.min(max_budget, math.floor(budget * grow))

194 stats.grows = stats.grows + 1

195 streak = 0

196 log(string.format("steady; raising budget -> %d", budget))

197 end

198 if delay > 0 then sleep(delay) end

199 end

200

201 stats.final_budget = budget

202 return stats

203end

204-- }}}