libs/mastodon-typed-text.lua

162 lines

1-- mastodon-typed-text.lua
2-- Reconstructs what an author typed into the Mastodon compose box from the
3-- rendered HTML stored in an ActivityPub archive, and counts that text the
4-- way the compose box counts it. The archive stores what the server RENDERED,
5-- not what was typed: markdown emphasis delimiters are consumed into tags
6-- (*love* becomes <em>love</em>), so a naive tag-strip silently loses those
7-- characters. This module puts the delimiters back before stripping tags,
8-- which is both a display-fidelity fix and the heart of golden poem (exactly
9-- 1024 characters as composed) qualification. See issue 4-003.
10
11local M = {}
12
13-- {{{ local function restore_delimiters
14-- Puts back the typed markdown delimiters that server-side rendering consumed.
15-- Plain-string gsubs (no pattern magic beyond the literal <>) so "<b>" cannot
16-- accidentally match "<br>" and "<s>" cannot match "<span>". Underscore-style
17-- emphasis (_x_) is indistinguishable from asterisk-style in the rendered HTML,
18-- so everything restores as asterisks: identical length, near-identical intent.
19local DELIMITER_RESTORATIONS = {
20 { "<strong>", "**" }, { "</strong>", "**" },
21 { "<b>", "**" }, { "</b>", "**" },
22 { "<em>", "*" }, { "</em>", "*" },
23 { "<i>", "*" }, { "</i>", "*" },
24 { "<del>", "~~" }, { "</del>", "~~" },
25 { "<s>", "~~" }, { "</s>", "~~" },
26 { "<code>", "`" }, { "</code>", "`" },
27}
28
29local function restore_delimiters(html)
30 for _, pair in ipairs(DELIMITER_RESTORATIONS) do
31 -- <, /, > are not Lua-pattern magic characters, and * is only special
32 -- in patterns (never in the replacement), so plain gsub is exact here
33 html = html:gsub(pair[1], pair[2])
34 end
35 return html
36end
37-- }}}
38
39-- {{{ function M.restore
40-- HTML from the archive in, reconstructed typed text out. Used for both the
41-- displayed poem content and the golden poem content.
42function M.restore(html)
43 local text = restore_delimiters(html)
44
45 -- paragraph and line-break structure back to the newlines that were typed
46 text = text:gsub("<p>", "\n\n")
47 -- all BR variants (<br>, <br/>, <br />): Mastodon emits XHTML-style <br />
48 text = text:gsub("<br%s*/?>", "\n")
49
50 -- entity decoding: specific entities FIRST, &amp; LAST. Decoding &amp;
51 -- first would turn a typed "&lt;" (stored as "&amp;lt;") into a bare "<"
52 -- by decoding it twice.
53 text = text:gsub("&lt;", "<")
54 text = text:gsub("&gt;", ">")
55 text = text:gsub("&quot;", "\"")
56 text = text:gsub("&#39;", "'")
57 text = text:gsub("&apos;", "'")
58 text = text:gsub("&amp;", "&")
59
60 -- legacy mojibake repairs carried over from the original cleaner: specific
61 -- observed damage to ^_^ emoticons in this archive. Length-neutral.
62 text = text:gsub(" _^", "^_^")
63 text = text:gsub("^^_^", "^_^")
64
65 -- NOTE: the original cleaner also deleted backslashes before quotes
66 -- (gsub('\\"', '"')). That destroyed typed text: 15 archived poems contain
67 -- intentional programming-style \" sequences ("the \"or\" operator").
68 -- Deliberately NOT reproduced here. See issue 4-003, August 2026.
69
70 -- everything not already translated is markup the author never typed
71 text = text:gsub("<[^>]+>", "")
72
73 -- the first <p> opens the text with newlines the author never typed;
74 -- trailing newlines cannot survive Mastodon's own posting whitespace-strip
75 text = text:gsub("^\n+", ""):gsub("\n+$", "")
76 return text
77end
78-- }}}
79
80-- {{{ function M.composer_length
81-- Length of a UTF-8 string as the compose box counter reported it to the
82-- author: one per character regardless of byte width (a curly quote is 3
83-- bytes but counted 1), one per emoji even from the astral plane, and zero
84-- for the invisible glue codepoints (variation selectors, zero-width joiner)
85-- that emoji pickers attach -- the author saw one heart, the counter charged
86-- one. Empirically anchored: two archived poems sit at exactly 1024 under
87-- this model and at 1025 under UTF-16-unit counting, and nothing above 1024
88-- could be typed into the box at all. Approximates grapheme clustering;
89-- multi-person ZWJ emoji sequences may still count their visible components.
90function M.composer_length(s)
91 local count, i, len = 0, 1, #s
92 while i <= len do
93 local b = s:byte(i)
94 if b < 0x80 then
95 i = i + 1
96 count = count + 1
97 elseif b < 0xE0 then
98 i = i + 2
99 count = count + 1
100 elseif b < 0xF0 then
101 -- 3-byte char: decode enough to spot the invisible glue
102 local b2, b3 = s:byte(i + 1), s:byte(i + 2)
103 local cp = (b % 0x10) * 0x1000 + (b2 % 0x40) * 0x40 + (b3 % 0x40)
104 local invisible = (cp >= 0xFE00 and cp <= 0xFE0F) -- variation selectors
105 or cp == 0x200D -- zero-width joiner
106 i = i + 3
107 if not invisible then
108 count = count + 1
109 end
110 else
111 i = i + 4
112 count = count + 1 -- astral emoji: one visible character, count 1
113 end
114 end
115 return count
116end
117-- }}}
118
119-- {{{ local function price_anchors
120-- Applies the compose box's link accounting before tags are stripped.
121-- Mention anchors (class="u-url mention") flatten to their visible "@user"
122-- text, which is exactly what the compose box charged for a typed
123-- @user@domain. Hashtag anchors keep their visible "#tag" text. Every other
124-- anchor is a URL, and the compose box charges a flat 23 characters no
125-- matter how long the URL is -- so the whole anchor (including the invisible
126-- spans holding the untruncated URL) becomes a 23-character placeholder.
127local URL_PLACEHOLDER = string.rep("x", 23)
128
129local function price_anchors(html)
130 return html:gsub("<a%s([^>]*)>(.-)</a>", function(attrs, inner)
131 if attrs:find("mention", 1, true) or attrs:find("hashtag", 1, true) then
132 return inner -- visible text survives; tags inside stripped later
133 end
134 return URL_PLACEHOLDER
135 end)
136end
137-- }}}
138
139-- {{{ function M.compose_box_count
140-- The number the author watched while composing: reconstructed typed text
141-- plus content warning text, in composer characters, with URLs priced at 23
142-- and mentions at their local part. Golden poems are the ones where this
143-- reached exactly 1024.
144function M.compose_box_count(html, content_warning)
145 local text = M.restore(price_anchors(html))
146
147 -- plain-text mentions the server never linkified (@user@domain typed in
148 -- the box) are still priced at @user by the compose counter; the domain
149 -- part rides free. Anchor-form mentions already flattened to bare @user.
150 -- Character set mirrors the privacy code's mention pattern.
151 text = text:gsub("@([%w%.%-_]+)@[%w%.%-]+%.%w+", "@%1")
152
153 local count = M.composer_length(text)
154 if content_warning and content_warning ~= "" then
155 count = count + M.composer_length(content_warning)
156 end
157 return count
158end
159-- }}}
160
161return M
162