scripts/test-model-propagation.sh

92 lines

1#!/usr/bin/env bash

2# test-model-propagation.sh -- pins the fix for the silent-model-fallback bug.

3# Run: bash scripts/test-model-propagation.sh

5# The bug: run.sh launches a fresh luajit per stage, and a --model override only

6# reached the stages run.sh explicitly threaded it through; the HTML, word-cloud

7# and word-page stages resolved the model via get_selected_model() /

8# embeddings_dir() with no argument and so silently reverted to config.lua's

9# default. The fix records the run's --model on a notepad in RAM

10# (tmp/run-overrides.lua) that the resolver consults. These checks lock in that

11# (a) an override propagates to the no-argument resolution every stage uses, in a

12# fresh process; (b) no override falls back to config.lua; (c) the notepad is

13# rewritten (never appended) so a previous run's choice cannot leak in.

15set -u

16DIR="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"

18"${DIR}/scripts/ensure-tmp-symlink" "${DIR}" >/dev/null || {

19 echo "FATAL: could not materialize tmp/ -- cannot test"; exit 2

20}

22pass=0; fail=0

23check() { # check <label> <actual> <expected>

24 if [ "$2" = "$3" ]; then pass=$((pass+1)); echo " ok - $1"

25 else fail=$((fail+1)); echo " FAIL - $1: got [$2] want [$3]"; fi

26}

28# {{{ resolve(): what a brand-new stage process sees as the selected model

29# Each call is its own luajit process, exactly like a real pipeline stage, so a

30# stale module cache cannot mask a propagation failure.

31resolve() {

32 luajit -e "

33 package.path = '${DIR}/libs/?.lua;${DIR}/src/?.lua;' .. package.path

34 local inf = require('inference-server-config')

35 inf.set_project_root('${DIR}')

36 io.write(inf.get_selected_model())

37 "

38}

39resolve_dir() { # the no-argument embeddings_dir() that stages 9/10 actually call

40 luajit -e "

41 package.path = '${DIR}/libs/?.lua;${DIR}/src/?.lua;' .. package.path

42 local u = require('utils'); u.init_assets_root({'${DIR}'})

43 io.write(u.embeddings_dir())

44 "

45}

46# }}}

48# {{{ config default (the value a no-override run must fall back to)

49CONFIG_DEFAULT="$(luajit -e "

50 package.path = '${DIR}/libs/?.lua;' .. package.path

51 local inf = require('inference-server-config'); inf.set_project_root('${DIR}')

52 io.write(inf.get_selected_model())

53")"

54# Read straight from config.lua independently, so this test does not depend on

55# the very resolver it is checking to define "correct".

56CONFIG_RAW="$(luajit -e "

57 local c = dofile('${DIR}/config.lua')

58 -- inference_servers is an array of {name=...}; the default is selected by name.

59 for _, s in ipairs(c.inference_servers) do

60 if s.name == c.default_inference_server then io.write(s.model); break end

61 end

62")"

63check "resolver default == config.lua default" "$CONFIG_DEFAULT" "$CONFIG_RAW"

64# }}}

66# {{{ an explicit --model propagates to the no-argument resolution, fresh process

67"${DIR}/scripts/write-run-overrides" "${DIR}" --model "propagation-probe:1b"

68check "get_selected_model honors override" "$(resolve)" "propagation-probe:1b"

69check "embeddings_dir(nil) keys off override" \

70 "$(resolve_dir)" "${DIR}/tmp/cache/embeddings/propagation-probe_1b"

71# }}}

73# {{{ no --model falls back to config.lua (override absent, not a hardcoded value)

74"${DIR}/scripts/write-run-overrides" "${DIR}" --model ""

75check "no override -> config default" "$(resolve)" "$CONFIG_RAW"

76# }}}

78# {{{ notepad is rewritten, not appended: a prior run's choice cannot leak in

79"${DIR}/scripts/write-run-overrides" "${DIR}" --model "first-run:9b"

80"${DIR}/scripts/write-run-overrides" "${DIR}" --model "second-run:9b"

81check "second write overwrites first" "$(resolve)" "second-run:9b"

82occurrences="$(grep -c "first-run" "${DIR}/tmp/run-overrides.lua" || true)"

83check "no stale value from prior run remains" "$occurrences" "0"

84# }}}

86# Leave the notepad clean (no override) so a later manual run is not surprised.

87"${DIR}/scripts/write-run-overrides" "${DIR}" --model "" >/dev/null

89echo ""

90echo "Result: ${pass} passed, ${fail} failed"

91[ "$fail" -eq 0 ]