libs/inference-server-config.lua

518 lines

1-- {{{ inference-server-config.lua

2-- Issue 10-049: Inference-server configuration loader (originally written for

3-- Ollama under 10-017; renamed and reframed for llama.cpp). Reads server

4-- definitions from config.lua and provides an API for server selection. The

5-- public surface intentionally stays close to the pre-migration shape so

6-- existing call sites in the rest of the codebase keep their structure.

7--

8-- Usage:

9-- local inference = require("inference-server-config")

10-- inference.set_project_root("/path/to/project") -- Required before other calls

11--

12-- -- Get servers

13-- local servers = inference.get_servers()

14-- local server = inference.get_server_by_name("gpu-server")

15-- local default = inference.get_default_server()

16--

17-- -- Build URL

18-- local url = inference.build_host_url(server) -- "http://192.168.0.115:10265"

19--

20-- -- Validate connection

21-- local ok, msg = inference.validate_server(server)

22-- }}}

24local M = {}

26-- {{{ Module state

27local project_root = nil

28local config = nil

29local selected_server = nil -- CLI override

30local selected_model = nil -- CLI override

32-- Whether the caller is in an interactive context. Off by default. The only

33-- way to enable it is for a CLI driver to call set_interactive_mode(true)

34-- after detecting -I on its command line — this is deliberately not a

35-- config.lua key, because the user-editable config file should describe

36-- what the project IS, not how the operator happens to be running it today.

37--

38-- The library applies a consistent policy whenever user input fails to

39-- resolve against a configured set of options (a typo in --server, an

40-- unrecognized --model, a missing default that points at a nonexistent

41-- entry, etc.): non-interactive callers hard-error immediately so the

42-- mistake is impossible to miss; interactive callers prompt the user to

43-- choose between using a sensible default or aborting. Silent fallback to

44-- a default is never the answer here — warnings get scrolled past in long

45-- log streams, and the wrong default can produce hours of work against the

46-- wrong endpoint before anyone notices.

47local interactive_mode = false

48-- }}}

50-- {{{ set_project_root

51-- Set the project root directory (required before loading config)

52function M.set_project_root(path)

53 project_root = path

54 config = nil -- Reset config when root changes

55end

56-- }}}

58-- {{{ local function load_config

59-- Load config.lua if not already loaded

60local function load_config()

61 if config then

62 return config

63 end

65 if not project_root then

66 -- Try to detect from package.path

67 local path = package.path:match("([^;]+)/libs/%?%.lua")

68 if path then

69 project_root = path

70 else

71 -- Fallback default

72 project_root = "/mnt/mtwo/programming/ai-stuff/neocities-modernization"

73 end

74 end

76 local config_path = project_root .. "/config.lua"

77 local ok, result = pcall(dofile, config_path)

78 if not ok then

79 -- Config not available, use empty

80 config = {}

81 return config

82 end

84 config = result

85 return config

86end

87-- }}}

89-- {{{ local function prompt_for_server_fallback

90-- Interactive recovery for "--server=<name> did not resolve."

91-- Shows the configured default server and asks whether to use it or abort.

92-- Prompts go to stderr so callers that capture stdout still surface them

93-- (though callers that capture stdout should not enable interactive mode

94-- in the first place). On success, also caches the choice by clearing

95-- selected_server, so a stage that calls get_selected_server twice does

96-- not re-prompt the operator.

97local function prompt_for_server_fallback(bad_name)

98 local cfg = load_config()

99 local default_name = cfg.default_inference_server

100 local default_server = default_name and M.get_server_by_name(default_name) or nil

101

102 io.stderr:write(string.format(

103 "\n[!] Inference server '%s' was not found in config.lua's inference_servers.\n", bad_name))

104

105 if not default_server then

106 io.stderr:write(" No usable default_inference_server is configured to fall back to.\n")

107 error(string.format(

108 "inference-server-config: --server=%s did not resolve and no default is available.", bad_name))

109 end

110

111 io.stderr:write("\nThe configured default is:\n")

112 io.stderr:write(string.format(" name: %s\n", default_server.name))

113 io.stderr:write(string.format(" host: %s\n", default_server.host or "(missing in config!)"))

114 io.stderr:write(string.format(" port: %s\n", tostring(default_server.port or "(missing in config!)")))

115 io.stderr:write(string.format(" model: %s\n", default_server.model or "(missing in config!)"))

116 io.stderr:write("\n 1) Use the default\n")

117 io.stderr:write(" 2) Error and exit\n")

118 io.stderr:write("\nSelect 1 or 2: ")

119

120 local choice = io.read("*l")

121 if choice == "1" then

122 selected_server = nil -- so subsequent calls go straight to the default without re-prompting

123 return default_server

124 end

125

126 error(string.format(

127 "inference-server-config: aborted by user — '%s' did not resolve and user chose to exit.", bad_name))

128end

129-- }}}

130

131-- {{{ function M.set_interactive_mode

132-- Flip the library into interactive mode. Only the CLI driver should call

133-- this, and only after confirming -I was passed on the command line.

134-- See the doc-comment on interactive_mode above for the policy this enables.

135function M.set_interactive_mode(enabled)

136 interactive_mode = enabled and true or false

137end

138-- }}}

139

140-- {{{ get_servers

141-- Get all configured Inference servers

142-- Returns array of server objects, or default fallback if none configured

143function M.get_servers()

144 local cfg = load_config()

145 local servers = cfg.inference_servers

146

147 if servers and #servers > 0 then

148 return servers

149 end

150

151 -- Fallback default if no servers configured. host:port matches the

152 -- operator's LAN-accessible llama.cpp box (192.168.1.100:10265) so a

153 -- bare-config run still resolves to the right endpoint.

154 return {

155 {

156 name = "local",

157 description = "Local llama.cpp instance (fallback)",

158 host = "192.168.1.100",

159 port = 10265,

160 model = "nomic-embed-text-v1.5"

161 }

162 }

163end

164-- }}}

165

166-- {{{ get_server_by_name

167-- Get a specific server by name

168-- Returns server object or nil if not found

169function M.get_server_by_name(name)

170 if not name then return nil end

171

172 for _, server in ipairs(M.get_servers()) do

173 if server.name == name then

174 return server

175 end

176 end

177

178 return nil

179end

180-- }}}

181

182-- {{{ get_default_server

183-- Resolve the configured default Inference server.

184--

185-- This is the "must work" path: callers that need an endpoint to make a

186-- request rely on this. It errors loudly if either default_inference_server

187-- is not set, or the named server does not exist in inference_servers.

188--

189-- A silent fallback to servers[1] used to live here. It was removed because

190-- it masked config drift: if default_inference_server was renamed without

191-- updating its referent, every consumer in the pipeline would silently

192-- start talking to whatever server happened to be first in the list,

193-- producing wrong results without any error message. Loud failure now

194-- guarantees that "endpoint resolution succeeded" means "your config said

195-- to use this server", not "we guessed."

196function M.get_default_server()

197 local cfg = load_config()

198

199 if not cfg.default_inference_server then

200 error("inference-server-config: config.lua does not set default_inference_server. "

201 .. "Set it to one of the names in inference_servers, or pass --server=<name> on the CLI.")

202 end

203

204 local server = M.get_server_by_name(cfg.default_inference_server)

205 if not server then

206 error(string.format(

207 "inference-server-config: default_inference_server is '%s' but no entry with that name exists in inference_servers. "

208 .. "Fix the name in config.lua, or add a matching inference_servers entry.",

209 cfg.default_inference_server))

210 end

211

212 return server

213end

214-- }}}

215

216-- {{{ get_selected_server

217-- Resolve the currently selected server.

218--

219-- Resolution order:

220-- 1. If --server=<name> was passed via set_selected_server, look it up

221-- in inference_servers.

222-- - If the name resolves, return that server.

223-- - If the name does not resolve:

224-- interactive: prompt the user to choose default or exit.

225-- non-interactive: hard-error with a message that names the

226-- offending --server=<name> and points at the fix.

227-- 2. If this run recorded a --server on the shared notepad, use that

228-- (Issue 10-065; see the block below for why).

229-- 3. Otherwise delegate to get_default_server, which either returns a

230-- resolved default or errors loudly if the default itself is missing

231-- or unresolvable.

232--

233-- This function deliberately never falls back silently. A typoed --server

234-- used to print a stderr warning and continue against the default, which

235-- meant a busy operator could miss the warning in the log stream and

236-- spend hours of pipeline time talking to the wrong endpoint.

237function M.get_selected_server()

238 if selected_server then

239 local server = M.get_server_by_name(selected_server)

240 if server then

241 return server

242 end

243

244 if interactive_mode then

245 return prompt_for_server_fallback(selected_server)

246 end

247

248 error(string.format(

249 "inference-server-config: --server=%s does not match any entry in inference_servers (config.lua).\n"

250 .. "Fix the name on the CLI, add a matching entry to inference_servers, "

251 .. "or pass -I to enable interactive selection.",

252 selected_server))

253 end

254

255 -- Issue 10-065: read the run's notepad, mirroring exactly what

256 -- get_selected_model does one function below. Without this, a --server

257 -- reached only the children run.sh remembered to hand it to on argv, and

258 -- every other child silently resolved default_inference_server instead.

259 --

260 -- Why that mattered more than "wrong hostname": a server entry also carries

261 -- embedding_prompt_prefix, the text prepended to every input before it is

262 -- embedded (see format_embedding_prompt). So the server choice selects an

263 -- embedding SPACE, not just an endpoint. The live symptom was inside a

264 -- single stage: run.sh passed --server to the poem-embedding step but

265 -- src/generate-word-pages.lua, which has no --server flag, resolved the

266 -- config default -- so poem and word embeddings could be produced against

267 -- different hosts with different prefixes, and the word pages then ranked

268 -- words against poems by cosine similarity across two different spaces.

269 --

270 -- load_config() first: it is what resolves project_root, which the notepad

271 -- reader needs in order to find tmp/shared-memory/run-overrides.lua.

272 load_config()

273 local overrides = require("runtime-overrides")

274 overrides.set_project_root(project_root)

275 local override_server = overrides.get("server")

276 if override_server then

277 local server = M.get_server_by_name(override_server)

278 if server then

279 return server

280 end

281 -- A name on the notepad that does not resolve gets the same hard error

282 -- a bad --server gets, and for the same reason: the notepad IS this

283 -- run's --server. Falling through to the config default here would

284 -- reintroduce the silent substitution this whole function refuses.

285 error(string.format(

286 "inference-server-config: this run recorded --server=%s, which does not match "

287 .. "any entry in inference_servers (config.lua).\n"

288 .. "Fix the name on the run.sh command line, or add a matching entry.",

289 override_server))

290 end

291

292 return M.get_default_server()

293end

294-- }}}

295

296-- {{{ set_selected_server

297-- Set the selected server name (from CLI --server flag)

298function M.set_selected_server(name)

299 selected_server = name

300end

301-- }}}

302

303-- {{{ get_selected_model

304-- Resolve the model identifier to send to the inference server.

305--

306-- The library does not validate --model=<name> against any local list.

307-- The inference server is the source of truth for "what model is loaded"

308-- — the config can only ever guess. If the operator passes a --model

309-- that the server does not have, the server returns a "model not found"

310-- error and the pipeline halts there. We deliberately do not want two

311-- layers both claiming to be authoritative about model existence; that

312-- produces drift bugs where the config lists models that are no longer

313-- installed, or omits models that are.

314--

315-- The available_models field on each inference_servers entry is still

316-- useful documentation for operators (and for list_servers' --list-servers

317-- output), it is just not consulted here as a gate.

318--

319-- Resolution order:

320-- 1. Resolve the server (delegates to get_selected_server, which errors

321-- or prompts if --server=<name> did not resolve).

322-- 2. If --model=<name> was passed, return it verbatim.

323-- 3. Otherwise return server.model. If that field is missing in the

324-- inference_servers entry, hard-error — config.lua is still the source

325-- of truth for "what model do we use by default on this host."

326function M.get_selected_model()

327 local server = M.get_selected_server()

328

329 if selected_model then

330 return selected_model

331 end

332

333 -- Model-propagation fix: a --model passed to run.sh is recorded once, at

334 -- startup, on the shared per-run notepad (tmp/run-overrides.lua). Consulting

335 -- it here means EVERY short-lived child process -- the HTML, word-cloud and

336 -- word-page stages that call this (or embeddings_dir() with no argument) --

337 -- resolves the SAME model the embedding stage used, instead of silently

338 -- reverting to server.model below. An absent notepad / absent key returns

339 -- nil, so a plain run (no --model) still falls through to config.lua exactly

340 -- as before. project_root is already resolved by get_selected_server above.

341 local overrides = require("runtime-overrides")

342 overrides.set_project_root(project_root)

343 local override_model = overrides.get("model")

344 if override_model then

345 return override_model

346 end

347

348 if not server.model then

349 error(string.format(

350 "inference-server-config: server '%s' has no 'model' field in config.lua's inference_servers entry. "

351 .. "Add a model = \"<name>\" field to that entry, or pass --model=<name> on the CLI.",

352 server.name))

353 end

354 return server.model

355end

356-- }}}

357

358-- {{{ set_selected_model

359-- Set the selected model (from CLI --model flag)

360function M.set_selected_model(model)

361 selected_model = model

362end

363-- }}}

364

365-- {{{ build_host_url

366-- Build the full URL for a server. Both host and port must be set in

367-- config.lua — a server entry without them is a config bug, not a chance

368-- for the library to guess sensible defaults. The previous code silently

369-- substituted "localhost" and 11434, which meant a forgotten host = in

370-- the config would silently redirect every embedding request to nothing.

371-- Returns URL string like "http://192.168.0.115:10265".

372function M.build_host_url(server)

373 if not server then

374 server = M.get_selected_server()

375 end

376

377 local name = server.name or "(unnamed server)"

378 if not server.host then

379 error(string.format(

380 "inference-server-config: server '%s' has no 'host' field in config.lua. "

381 .. "Add a host = \"<hostname-or-ip>\" field to that inference_servers entry.", name))

382 end

383 if not server.port then

384 error(string.format(

385 "inference-server-config: server '%s' has no 'port' field in config.lua. "

386 .. "Add a port = <number> field to that inference_servers entry.", name))

387 end

388

389 return string.format("http://%s:%d", server.host, server.port)

390end

391-- }}}

392

393-- {{{ validate_server

394-- Check if a server is reachable

395-- Returns: success (bool), message (string)

396function M.validate_server(server)

397 if not server then

398 server = M.get_selected_server()

399 end

400

401 -- /v1/models is llama.cpp's OpenAI-compatible "what's loaded" endpoint.

402 -- Was /api/tags under Ollama; migrated in 10-049 along with the rest

403 -- of the API surface.

404 local url = M.build_host_url(server) .. "/v1/models"

405 local cmd = string.format("curl -s -o /dev/null -w '%%{http_code}' --max-time 3 '%s' 2>/dev/null", url)

406

407 local handle = io.popen(cmd)

408 local status = handle:read("*a")

409 handle:close()

410

411 status = status:gsub("%s+", "") -- Trim whitespace

412

413 if status == "200" then

414 return true, "Server is reachable"

415 elseif status == "000" then

416 return false, "Connection timeout - server unreachable"

417 else

418 return false, "Server returned HTTP " .. status

419 end

420end

421-- }}}

422

423-- {{{ list_servers

424-- Print a formatted list of available servers.

425-- Reads default_inference_server directly rather than calling get_default_server

426-- so that --list-servers works even when no default is configured (or is

427-- misconfigured). The purpose of this function is diagnostic, not

428-- request-issuing — it must not error when the user is trying to inspect

429-- their config.

430function M.list_servers()

431 local cfg = load_config()

432 local servers = M.get_servers()

433 local default_name = cfg.default_inference_server -- may be nil; that's fine here

434

435 print("Available Inference servers:")

436 print(string.rep("-", 70))

437

438 for _, server in ipairs(servers) do

439 local is_default = (default_name ~= nil and server.name == default_name)

440 local default_marker = is_default and " (default)" or ""

441 local url = M.build_host_url(server)

442

443 print(string.format(" %s%s", server.name, default_marker))

444 print(string.format(" %s", server.description or ""))

445 print(string.format(" URL: %s", url))

446 print(string.format(" Model: %s", server.model or "nomic-embed-text"))

447

448 if server.available_models and #server.available_models > 0 then

449 -- available_models entries may be plain strings or {model=...} tables

450 -- (a table also carries its own GGUF + prompt); show just the names.

451 local names = {}

452 for _, entry in ipairs(server.available_models) do

453 names[#names + 1] = (type(entry) == "table") and entry.model or entry

454 end

455 print(string.format(" Available models: %s", table.concat(names, ", ")))

456 end

457 print("")

458 end

459end

460-- }}}

461

462-- {{{ format_embedding_prompt

463-- Apply the active server's embedding_prompt_prefix to a text payload.

464--

465-- Some embedding models (notably nomic-embed-text v1.5+) require a

466-- task-prefix on every input — "clustering: ", "search_query: ",

467-- "search_document: ", etc. — that routes the model through different

468-- internal weights. Models that don't need a prefix (embeddinggemma,

469-- qwen3-embedding) leave the field nil and this function is a no-op.

470--

471-- Centralizing the prefix here means a model swap is a single config

472-- edit even when the new model has different prefix requirements; no

473-- caller needs to know which model is active to embed text correctly.

474function M.format_embedding_prompt(text)

475 local cfg = M.get_selected_model_config()

476 local prefix = cfg and cfg.embedding_prompt_prefix

477 if prefix and prefix ~= "" then

478 return prefix .. text

479 end

480 return text

481end

482-- }}}

483

484-- {{{ get_selected_model_config

485-- Resolve { model, model_path, embedding_prompt_prefix } for the SELECTED model

486-- on the selected server. This is what lets one server entry serve several local

487-- GGUFs: each available_models entry may be a table carrying its own model_path

488-- and prompt prefix, so `--server local --model X` loads X with X's phrasing.

489--

490-- Resolution: if the selected model matches a TABLE entry in available_models,

491-- use that entry's fields. Otherwise (a plain-string entry, or a server whose

492-- available_models is documentation-only like the remote gpu-server) fall back

493-- to the server's top-level model_path / embedding_prompt_prefix -- the default

494-- model. So the common case (no --model, default model) is unchanged.

495function M.get_selected_model_config()

496 local server = M.get_selected_server()

497 local model = M.get_selected_model()

498 if server.available_models then

499 for _, entry in ipairs(server.available_models) do

500 if type(entry) == "table" and entry.model == model then

501 return {

502 model = model,

503 model_path = entry.model_path or server.model_path,

504 embedding_prompt_prefix = entry.embedding_prompt_prefix,

505 }

506 end

507 end

508 end

509 return {

510 model = model,

511 model_path = server.model_path,

512 embedding_prompt_prefix = server.embedding_prompt_prefix,

513 }

514end

515-- }}}

516

517return M

518