libs/sources-loader.lua

1-- {{{ sources-loader.lua

2-- Issue 10-015: Unified input sources configuration loader

3-- Issue 10-026: Extended with external sync support (merged external_files)

4--

5-- Reads from config.sources and provides iteration over source directories.

6-- Also provides access to external sync information (where to pull data from).

7--

8-- The sources config supports multiple named directories per source type,

9-- enabling deduplication and flexible input management.

10--

11-- Usage:

12-- local loader = require("sources-loader")

13-- loader.set_project_root("/path/to/project")

14--

15-- -- Get all directories for a source type

16-- local fedi_dirs = loader.get_directories("fediverse")

17-- for _, dir in ipairs(fedi_dirs) do

18-- print(dir.name, dir.path)

19-- end

20--

21-- -- Check if a source is enabled

22-- if loader.is_enabled("bluesky") then ... end

23--

24-- -- Get source metadata

25-- local src = loader.get_source("fediverse")

26-- print(src.format) -- "activitypub"

27--

28-- -- Get all external sync entries (Issue 10-026)

29-- local syncs = loader.get_all_external_syncs()

30-- for _, sync in ipairs(syncs) do

31-- print(sync.name, sync.source, sync.destination)

32-- end

33--

34-- -- Get archives for a source type

35-- local archives = loader.get_archives("fediverse")

36-- }}}

40-- {{{ Module state

41local project_root = nil

42local config = nil

43-- }}}

45-- {{{ ANSI color codes for terminal output

46local COLOR_GREEN = "\027[92m"

47local COLOR_RED = "\027[91m"

48local COLOR_YELLOW = "\027[93m"

49local COLOR_RESET = "\027[0m"

50-- }}}

52-- {{{ set_project_root

53-- Set the project root directory (required before any operations)

54function M.set_project_root(path)

55 project_root = path

56 config = nil -- Reset config when root changes

57end

58-- }}}

60-- {{{ local function load_config

61-- Load config.lua if not already loaded

62local function load_config()

63 if config then

64 return config

65 end

66

67 if not project_root then

68 error("sources-loader: project_root not set. Call set_project_root() first.")

69 end

70

71 local config_path = project_root .. "/config.lua"

72 local ok, result = pcall(dofile, config_path)

73 if not ok then

74 error("sources-loader: Failed to load config.lua: " .. tostring(result))

75 end

76

77 config = result

78 return config

79end

80-- }}}

82-- {{{ local function get_sources

83-- Get the sources table from config

84local function get_sources()

85 local cfg = load_config()

86 return cfg.sources or {}

87end

88-- }}}

90-- {{{ local function resolve_path

91-- Resolve a path (make absolute if relative)

92local function resolve_path(path)

93 if not path then return nil end

94

95 -- If already absolute, return as-is

96 if path:sub(1, 1) == "/" then

97 return path

98 end

99

100 -- Relative to project root

101 return project_root .. "/" .. path

102end

103-- }}}

105-- {{{ local function dir_exists

106-- Check if a directory exists

107local function dir_exists(path)

108 local handle = io.popen("test -d '" .. path .. "' && echo yes || echo no")

109 local result = handle:read("*l")

110 handle:close()

111 return result == "yes"

112end

113-- }}}

115-- {{{ get_source

116-- Get a source configuration by type name

117-- Returns: table with source config or nil if not found

118function M.get_source(source_type)

119 local sources = get_sources()

120 return sources[source_type]

121end

122-- }}}

124-- {{{ is_enabled

125-- Check if a source type is enabled

126-- Returns: boolean

127function M.is_enabled(source_type)

128 local source = M.get_source(source_type)

129 if not source then

130 return false

131 end

132 -- Default to true if enabled field is missing

133 return source.enabled ~= false

134end

135-- }}}

137-- {{{ get_format

138-- Get the format string for a source type

139-- Returns: string or nil

140function M.get_format(source_type)

141 local source = M.get_source(source_type)

142 if not source then

143 return nil

144 end

145 return source.format

146end

147-- }}}

149-- {{{ get_directories

150-- Get all directories for a source type

151-- Returns: array of { name, path, description, randomize_order, random_seed }

152-- Paths are resolved to absolute paths

153-- Issue 10-030: Added randomize_order and random_seed fields for image position randomization

154function M.get_directories(source_type)

155 local source = M.get_source(source_type)

156 if not source then

157 return {}

158 end

159

160 local dirs = source.directories or {}

161 local result = {}

162

163 for _, dir in ipairs(dirs) do

164 table.insert(result, {

165 name = dir.name or "unnamed",

166 path = resolve_path(dir.path),

167 description = dir.description or "",

168 -- Issue 10-030: Randomization options for image sources

169 randomize_order = dir.randomize_order or false,

170 random_seed = dir.random_seed -- nil means use system random

171 })

172 end

173

174 return result

175end

176-- }}}

178-- {{{ get_valid_directories

179-- Get directories for a source type. Every configured directory is MANDATORY:

180-- the "optional" concept was removed deliberately -- a missing source means the

181-- data we expected to ship isn't there, which we want to know about loudly rather

182-- than silently skip. So this errors on the FIRST missing directory.

183-- Returns: array of directories on success, or nil + error message on failure.

184function M.get_valid_directories(source_type)

185 local dirs = M.get_directories(source_type)

186 local valid = {}

187

188 for _, dir in ipairs(dirs) do

189 if dir_exists(dir.path) then

190 table.insert(valid, dir)

191 else

192 return nil, string.format(

193 "Required directory '%s' not found: %s",

194 dir.name, dir.path)

195 end

196 end

197

198 return valid

199end

200-- }}}

202-- {{{ get_source_types

203-- Get all configured source type names

204-- Returns: array of strings (e.g., {"fediverse", "messages", "notes", "bluesky", "images"})

205function M.get_source_types()

206 local sources = get_sources()

207 local types = {}

208

209 for name, _ in pairs(sources) do

210 table.insert(types, name)

211 end

212

213 -- Sort for consistent ordering

214 table.sort(types)

215 return types

216end

217-- }}}

219-- {{{ get_enabled_sources

220-- Get all enabled source types

221-- Returns: array of strings

222function M.get_enabled_sources()

223 local all_types = M.get_source_types()

224 local enabled = {}

225

226 for _, source_type in ipairs(all_types) do

227 if M.is_enabled(source_type) then

228 table.insert(enabled, source_type)

229 end

230 end

231

232 return enabled

233end

234-- }}}

236-- {{{ get_media_config

237-- Get media configuration for a source type (if any)

238-- Returns: table with media settings or nil

239function M.get_media_config(source_type)

240 local source = M.get_source(source_type)

241 if not source then

242 return nil

243 end

244 return source.media

245end

246-- }}}

248-- {{{ validate_source

249-- Validate a source configuration

250-- Returns: true, nil on success; false, error_message on failure

251function M.validate_source(source_type)

252 local source = M.get_source(source_type)

253

254 if not source then

255 return false, "Source type not found: " .. source_type

256 end

257

258 -- Check directories exist

259 if not source.directories or #source.directories == 0 then

260 return false, source_type .. ": No directories configured"

261 end

262

263 -- Check each directory has required fields

264 for i, dir in ipairs(source.directories) do

265 if not dir.path then

266 return false, string.format(

267 "%s: Directory #%d missing 'path' field",

268 source_type, i)

269 end

270 end

271

272 return true, nil

273end

274-- }}}

276-- {{{ validate_all

277-- Validate all source configurations

278-- Returns: { valid = bool, errors = {}, warnings = {} }

279function M.validate_all()

280 local result = {

281 valid = true,

282 errors = {},

283 warnings = {}

284 }

285

286 local source_types = M.get_source_types()

287

288 for _, source_type in ipairs(source_types) do

289 local ok, err = M.validate_source(source_type)

290 if not ok then

291 result.valid = false

292 table.insert(result.errors, err)

293 end

294

295 -- Check if enabled and directories exist

296 if M.is_enabled(source_type) then

297 local dirs, err = M.get_valid_directories(source_type)

298 if not dirs then

299 -- A missing source directory is now a hard failure: every source is

300 -- mandatory (the optional concept was removed), so absence is an error.

301 result.valid = false

302 table.insert(result.errors, err)

303 end

304 end

305 end

306

307 return result

308end

309-- }}}

311-- {{{ print_sources

312-- Print a formatted summary of configured sources

313function M.print_sources()

314 local source_types = M.get_source_types()

315

316 if #source_types == 0 then

317 print("No sources configured")

318 return

319 end

320

321 print("Configured sources:")

322 print(string.rep("-", 70))

323

324 for _, source_type in ipairs(source_types) do

325 local source = M.get_source(source_type)

326 local enabled = M.is_enabled(source_type)

327 local status = enabled and COLOR_GREEN .. "enabled" .. COLOR_RESET

328 or COLOR_YELLOW .. "disabled" .. COLOR_RESET

329

330 print(string.format(" %s [%s]", source_type, status))

331

332 if source.format then

333 print(string.format(" Format: %s", source.format))

334 end

335

336 local dirs = M.get_directories(source_type)

337 if #dirs > 0 then

338 print(" Directories:")

339 for _, dir in ipairs(dirs) do

340 local exists = dir_exists(dir.path)

341 local marker = exists and "✓" or "✗"

342 print(string.format(" %s %s: %s",

343 marker, dir.name, dir.path))

344 end

345 end

346 end

347end

348-- }}}

350-- {{{ iterate_directories

351-- Iterate over all valid directories for a source type, calling a function for each

352-- Errors on any missing directory (every source is mandatory)

353-- callback(dir) receives { name, path, description }

354-- Returns: true on success, false + error on failure

355function M.iterate_directories(source_type, callback)

356 if not M.is_enabled(source_type) then

357 return true -- Not enabled, nothing to do

358 end

359

360 local dirs, err = M.get_valid_directories(source_type)

361 if not dirs then

362 return false, err

363 end

364

365 for _, dir in ipairs(dirs) do

366 callback(dir)

367 end

368

369 return true

370end

371-- }}}

373-- {{{ get_directories_with_external

374-- Issue 10-026: Get directories including external sync information

375-- Returns: array of { name, path, description, external }

376-- external is { source = "...", destination = "..." } or nil

377function M.get_directories_with_external(source_type)

378 local source = M.get_source(source_type)

379 if not source then

380 return {}

381 end

382

383 local dirs = source.directories or {}

384 local result = {}

385

386 for _, dir in ipairs(dirs) do

387 local entry = {

388 name = dir.name or "unnamed",

389 path = resolve_path(dir.path),

390 description = dir.description or "",

391 external = nil

392 }

393

394 -- Include external sync info if present

395 -- external.source is where to pull from

396 -- destination is derived from path (strip "input/" prefix)

397 if dir.external and dir.external.source then

398 local dest = dir.path or ""

399 -- Strip "input/" prefix if present to get destination relative to input/

400 if dest:sub(1, 6) == "input/" then

401 dest = dest:sub(7)

402 end

403 entry.external = {

404 source = dir.external.source,

405 destination = dest

406 }

407 end

408

409 table.insert(result, entry)

410 end

411

412 return result

413end

414-- }}}

416-- {{{ get_archives

417-- Issue 10-026: Get archive entries for a source type

418-- Archives are ZIP files that need extraction rather than rsync

419-- Returns: array of { name, source, extract_to }

420function M.get_archives(source_type)

421 local source = M.get_source(source_type)

422 if not source then

423 return {}

424 end

425

426 local archives = source.archives or {}

427 local result = {}

428

429 for _, archive in ipairs(archives) do

430 table.insert(result, {

431 name = archive.name or "unnamed",

432 source = archive.source,

433 extract_to = resolve_path(archive.extract_to or "input")

434 })

435 end

436

437 return result

438end

439-- }}}

441-- {{{ get_all_external_syncs

442-- Issue 10-026: Get ALL external sync entries across all sources

443-- Returns data in external_files-compatible format for backward compatibility

444-- Returns: array of { name, source, destination, is_archive }

445function M.get_all_external_syncs()

446 local sources = get_sources()

447 local result = {}

448

449 for source_type, source in pairs(sources) do

450 -- Collect directory-based syncs (rsync)

451 if source.directories then

452 for _, dir in ipairs(source.directories) do

453 if dir.external and dir.external.source then

454 local dest = dir.path or ""

455 -- Strip "input/" prefix for destination

456 if dest:sub(1, 6) == "input/" then

457 dest = dest:sub(7)

458 end

459 table.insert(result, {

460 name = dir.name or "unnamed",

461 source = dir.external.source,

462 destination = dest,

463 is_archive = false,

464 source_type = source_type

465 })

466 end

467 end

468 end

469

470 -- Collect archive-based syncs (unzip)

471 if source.archives then

472 for _, archive in ipairs(source.archives) do

473 local dest = archive.extract_to or "input"

474 -- Strip "input/" prefix for destination

475 if dest:sub(1, 6) == "input/" then

476 dest = dest:sub(7)

477 end

478 -- Empty string means extract to input/ root

479 if dest == "input" then

480 dest = ""

481 end

482 table.insert(result, {

483 name = archive.name or "unnamed",

484 source = archive.source,

485 destination = dest,

486 is_archive = true,

487 source_type = source_type

488 })

489 end

490 end

491 end

492

493 return result

494end

495-- }}}

497-- {{{ has_external_syncs

498-- Issue 10-026: Check if any source has external sync configuration

499-- Returns: boolean

500function M.has_external_syncs()

501 local syncs = M.get_all_external_syncs()

502 return #syncs > 0

503end

504-- }}}