scripts/extract-notes.lua

254 lines

1-- Notes content extraction script

2-- Processes text files from notes directory and extracts formatted content

4-- {{{ setup_dir_path

5local function setup_dir_path(provided_dir)

6 if provided_dir then

7 return provided_dir

8 end

9 return "/mnt/mtwo/programming/ai-stuff/neocities-modernization"

10end

11-- }}}

13-- Get project directory from command line or use default

14local DIR = setup_dir_path(arg and arg[1])

15local OVERRIDE_SOURCE = arg and arg[2] -- Optional override for temporary extraction

17-- Set up package path to find libs

18package.path = DIR .. "/libs/?.lua;" .. package.path

19local dkjson = require("dkjson")

20local exclusion_filter = require("exclusion-filter")

22-- Issue 10-003: Load unified config from config.lua

23local config_loader = require("config-loader")

24config_loader.set_project_root(DIR)

25local config = config_loader.load()

27-- Issue 10-015: Load sources configuration for multi-directory support

28local sources_loader = require("sources-loader")

29sources_loader.set_project_root(DIR)

31-- ANSI color codes for terminal output

32local COLOR_GREEN = "\027[92m" -- Bright green for success (✓, ✅)

33local COLOR_BLUE = "\027[94m" -- Bright blue for info (ℹ️)

34local COLOR_RED = "\027[91m" -- Bright red for errors (✗, ❌)

35local COLOR_YELLOW = "\027[93m" -- Bright yellow for warnings (⚠️)

36local COLOR_RESET = "\027[0m" -- Reset to default

38-- {{{ local function relative_path

39-- Issue 7-003: Show project name instead of "./" when path equals DIR

40local function relative_path(absolute_path)

41 if absolute_path == DIR or absolute_path == DIR .. "/" then

42 local dir_name = DIR:match("([^/]+)/?$")

43 return dir_name .. "/"

44 end

45 if absolute_path:sub(1, #DIR) == DIR then

46 local rel = absolute_path:sub(#DIR + 1)

47 if rel:sub(1, 1) == "/" then rel = rel:sub(2) end

48 return "./" .. rel

49 end

50 return absolute_path

51end

52-- }}}

54-- Issue 10-015a: Get notes path from unified sources config (no fallback - errors if not configured)

55local notes_directories = sources_loader.get_directories("notes")

56if #notes_directories == 0 then

57 print(COLOR_RED .. "❌ Error: sources.notes not configured in config.lua" .. COLOR_RESET)

58 os.exit(1)

59end

60-- Use the primary directory from sources config

61local notes_backup_path = notes_directories[1].path

62-- Strip DIR prefix if present (sources-loader returns absolute paths)

63if notes_backup_path:sub(1, #DIR) == DIR then

64 notes_backup_path = notes_backup_path:sub(#DIR + 2) -- +2 for the slash

65end

67-- Use override path if provided (for ZIP extraction), otherwise use configured path

68local source_base_path

69if OVERRIDE_SOURCE then

70 source_base_path = OVERRIDE_SOURCE

71 print("🔄 Using temporary extraction source: " .. relative_path(source_base_path))

72else

73 source_base_path = DIR .. "/" .. notes_backup_path

74 print("🔄 Using configured source: " .. relative_path(source_base_path))

75end

77-- Set up paths

78local notes_dir = source_base_path

79local save_location = DIR .. "/" .. notes_backup_path .. "/files"

81-- {{{ function shell_escape

82local function shell_escape(str)

83 -- Escape single quotes for shell: ' becomes '\''

84 -- This ends the quote, adds an escaped quote, and starts a new quote

85 return "'" .. str:gsub("'", "'\\''") .. "'"

86end

87-- }}}

89-- {{{ function get_file_mtime

90local function get_file_mtime(file_path)

91 -- Use stat command to get modification time

92 local stat_cmd = string.format("stat -c %%Y %s 2>/dev/null", shell_escape(file_path))

93 local handle = io.popen(stat_cmd)

94 local result = handle:read("*a")

95 handle:close()

97 if result and result ~= "" then

98 local clean_result = result:gsub("%s+", "")

99 local timestamp = tonumber(clean_result)

100 if timestamp then

101 return timestamp

102 end

103 end

104

105 -- Fallback to current time if stat fails

106 return os.time()

107end

108-- }}}

109

110-- {{{ function format_iso_date

111local function format_iso_date(timestamp)

112 if type(timestamp) ~= "number" then

113 timestamp = os.time()

114 end

115 return os.date("%Y-%m-%dT%H:%M:%SZ", timestamp)

116end

117-- }}}

118

119-- {{{ function generate_poem_metadata

120local function generate_poem_metadata(content, file_path)

121 local mtime = get_file_mtime(file_path)

122

123 local metadata = {

124 character_count = string.len(content),

125 word_count = select(2, content:gsub("%S+", "")),

126 has_content_warning = false, -- Notes typically don't have explicit CW

127 extraction_timestamp = os.date("%Y-%m-%dT%H:%M:%SZ"),

128 source_file = file_path:match("([^/]+)$"), -- Just filename

129 file_modification_time = format_iso_date(mtime)

130 }

131

132 return metadata

133end

134-- }}}

135

136-- {{{ function is_valid_note_file

137local function is_valid_note_file(file_path)

138 -- Skip hidden files, backup files, and directories

139 local filename = file_path:match("([^/]+)$")

140 if not filename then return false end

141

142 -- Skip hidden files (starting with .)

143 if filename:match("^%.") then return false end

144

145 -- Skip backup files (.un~, .swp, etc.)

146 if filename:match("%.un~$") or filename:match("%.swp$") then return false end

147

148 -- Skip JSON files (including our own output)

149 if filename:match("%.json$") then return false end

150

151 -- Skip directories (check if it's a regular file)

152 local file_handle = io.open(file_path, "r")

153 if not file_handle then return false end

154 file_handle:close()

155

156 return true

157end

158-- }}}

159

160print("📝 Starting notes extraction from: " .. relative_path(notes_dir))

161

162-- Issue 6-031: Load poem exclusion filter

163-- For notes, exclusion IDs are filenames (without extension)

164local poem_exclusions = exclusion_filter.load_default(DIR)

165if poem_exclusions:count("notes") > 0 then

166 -- Issue 7-006: Full-line coloring for info messages

167 print(COLOR_YELLOW .. "🚫 Notes exclusion filter: " .. poem_exclusions:count("notes") .. " entries" .. COLOR_RESET)

168end

169

170local excluded_count = 0

171

172-- Scan notes directory for files (exclude files/ subdirectory to avoid reading our own output)

173local find_cmd = string.format("find %s -type f -not -path '*/files/*'", shell_escape(notes_dir))

174local find_handle = io.popen(find_cmd)

175

176local poems_json = {}

177local i = 1

178

179for file_path in find_handle:lines() do

180 if is_valid_note_file(file_path) then

181 local filename = file_path:match("([^/]+)$")

182

183 -- Issue 6-031: Notes use filename (without extension) as exclusion ID

184 -- This matches the human-readable format in config/excluded-poems.txt

185 local note_id = filename:match("(.+)%..+$") or filename

186 if poem_exclusions:is_excluded("notes", note_id) then

187 excluded_count = excluded_count + 1

188 goto continue

189 end

190

191 -- Read file content

192 local file_handle = io.open(file_path, "r")

193 if file_handle then

194 local content = file_handle:read("*a")

195 file_handle:close()

196

197 -- Clean up content (remove excessive whitespace)

198 content = content:gsub("\n\n+", "\n\n") -- Reduce multiple newlines

199 content = content:gsub("^%s+", ""):gsub("%s+$", "") -- Trim whitespace

200

201 if content and content ~= "" then

202 local mtime = get_file_mtime(file_path)

203

204 -- Generate JSON format for HTML generation

205 local poem_entry = {

206 id = string.format("%04d", i),

207 category = "notes",

208 source_file = filename,

209 creation_date = format_iso_date(mtime),

210 content_warning = nil, -- Notes typically don't have CW

211 content = content,

212 raw_content = content, -- Notes don't have markup

213 metadata = generate_poem_metadata(content, file_path)

214 }

215 table.insert(poems_json, poem_entry)

216 i = i + 1

217 end

218 end

219 end

220

221 ::continue::

222end

223

224find_handle:close()

225

226-- {{{ Generate JSON output for HTML generation

227-- Create output directory

228os.execute("mkdir -p " .. shell_escape(save_location))

229

230-- Generate JSON output

231local json_output = {

232 poems = poems_json,

233 extraction_summary = {

234 total_poems = #poems_json,

235 poems_excluded = excluded_count, -- Issue 6-031: Excluded poem count

236 by_category = { notes = #poems_json },

237 content_warnings = {}, -- Notes typically don't have content warnings

238 extraction_date = os.date("%Y-%m-%dT%H:%M:%SZ")

239 }

240}

241

242local json_file = save_location .. "/poems.json"

243local f = io.open(json_file, "w")

244f:write(dkjson.encode(json_output, { indent = true }))

245f:close()

246

247-- Issue 7-006: Full-line coloring for success messages

248print(COLOR_GREEN .. "✅ Notes extraction complete" .. COLOR_RESET)

249print(" 📄 Generated: " .. relative_path(json_file))

250print(" 📊 Notes processed: " .. #poems_json)

251if excluded_count > 0 then

252 print(" 🚫 Excluded: " .. excluded_count .. " (tombstoned)")

253end

254-- }}}