Koha Solutions knowledge base: run log of a live check Article https://kohasolutions.com/knowledge-base/koha-broken-characters-after-import Check scripts/kb/live/koha-broken-characters-after-import.mjs (SHA-256 28a3afd6f553e4836f18df791c6f617ae208ba7282be95df9edc82ef130d1306) Ran 2026-11-08T07:36:13Z (UTC), 85 s Koha 26.05.05 (Koha::version 26.05.05.000, database version 26.0505000) Source https://git.koha-community.org/Koha-community/Koha/src/commit/d3fc229bc5eaea25dfe88f86149386408fe5d32d System Ubuntu 24.04.5 LTS Setup A fresh Ubuntu 24.04 server with Koha installed from the Koha repository as our installation guide leaves it (yaz comes with koha-common), reached over SSH: every command run as the sudo user librarian on a terminal, in its home folder. Result PASS: all 13 claims held Each claim is something the article says. "seen" is what the check read back when it is not just a yes/no. Changes the check made to the test server were undone afterwards, as its header describes. == Claims == ok the server box starts with Koha installed ok koha-common Depends: yaz (yaz-marcdump is installed with Koha) ok yaz-marcdump is on the PATH of a normal user ok MARC-8 file: head -n 3 → "00155nam 2200061 a 4500" / "001 kb40-1" / "100 1 $a Garc�ia M�arquez, Gabriel." seen: "00155nam 2200061 a 4500\n001 kb40-1\n100 1 $a Garc�ia M�arquez, Gabriel." ok the MARC-8 → UTF-8 conversion prints nothing, exit 0 seen: {"out":"","code":0} ok converted: "00160nam a2200061 a 4500" / "001 kb40-1" / "100 1 $a García Márquez, Gabriel." (as displayed) seen: "00160nam a2200061 a 4500\n001 kb40-1\n100 1 $a García Márquez, Gabriel." ok …and the 245 reads "Cien años de soledad / $c Gabriel García Márquez." ok yaz writes the accented letters decomposed: "í" is "i" + U+0301 (combining acute), so it looks the same seen: " $a García Márquez, Gabriel." ok Windows-1256 has no U+06CC (Urdu ی) ok CP1256 file as UTF-8: "245 00 $a ����� ����" seen: "00072nam 2200049 a 4500\n001 kb40-2\n245 00 $a ����� ����" ok -f CP1256 is accepted, prints nothing seen: {"out":"","code":0} ok converted Urdu record: "00081nam a2200049 a 4500" / "001 kb40-2" / "245 00 $a ديوان غالب" seen: "00081nam a2200049 a 4500\n001 kb40-2\n245 00 $a ديوان غالب" ok …and holds the Arabic yeh U+064A, not U+06CC == The check that produced this log (scripts/kb/live/koha-broken-characters-after-import.mjs) == /** * Live check for prisma/ks-kb/40-koha-broken-characters-after-import.ts, the yaz-marcdump blocks. Does NOT touch live * Koha. Since 2026-11-08 (plan212) it runs on a real server: a fresh Ubuntu 24.04 with Koha installed as our * installation guide leaves it (koha-common Depends on yaz), reached over SSH as the sudo user `librarian`, in that * user's home folder, on a terminal. It makes a MARC-8 file from MARCXML with yaz-marcdump itself, and a Windows-1256 * Urdu record byte by byte (yaz cannot write CP1256), then runs the article's commands and asserts the responses it * shows. (2026-11-05: the same claims in a throwaway kbv-koha-ubuntu:24.04 container; that proof stays in * proofs.json.) The staging behaviour (�, the log-only warning) is from the 2026-11-04 run on live Koha recorded in the * article's header. The box is removed at the end. About 2 minutes once the image exists. */ import { createBox, ensureKohaImage, KOHA_IMAGE, removeBox, ssh } from './server-box/box.mjs' /** liveKoha: false — runs only on its own server box, so kb:verify neither starts nor reads live Koha. */ export const meta = { koha: '26.05.05', passed: '2026-11-05', liveKoha: false } const XML = ` 00000nam a2200000 a 4500 kb40-1 García Márquez, Gabriel. Cien años de soledad /Gabriel García Márquez. ` const PY = `FT, SF, RT = b"\\x1e", b"\\x1f", b"\\x1d" def rec(fields): data, dirs, pos = b"", b"", 0 for tag, body in fields: b = body + FT dirs += tag.encode() + b"%04d%05d" % (len(b), pos); pos += len(b); data += b base = 24 + len(dirs) + 1 return (b"%05dnam 22%05d a 4500" % (base + len(data) + 1, base)) + dirs + FT + data + RT open("urdu.mrc", "wb").write(rec([("001", b"kb40-2"), ("245", b"00" + SF + b"a" + "\\u062f\\u064a\\u0648\\u0627\\u0646 \\u063a\\u0627\\u0644\\u0628".encode("cp1256"))])) try: "\\u06cc".encode("cp1256"); print("06cc-in-cp1256") except UnicodeEncodeError: print("06cc-not-in-cp1256")` export default async function run(ctx) { const err = ensureKohaImage() ?? createBox({ image: KOHA_IMAGE }) ctx.check('the server box starts with Koha installed', err === null, err ?? '') if (err) return try { await body(ctx) } finally { removeBox() } } async function body(ctx) { const L = (cmd) => ssh('librarian', cmd).replace(/\s+$/, '') const put = (name, text) => L(`echo ${Buffer.from(text).toString('base64')} | base64 -d > ${name}`) // what the reader's terminal shows, and the command's exit status on the last line const u = (cmd) => { const out = L(`${cmd}; echo "exit=$?"`) const m = /\n?exit=(\d+)$/.exec(out) // a whole record ends in a blank line; a terminal shows nothing after it, so trailing blank lines are dropped return { out: (m ? out.slice(0, m.index) : out).replace(/\s+$/, ''), code: m ? Number(m[1]) : -1 } } ctx.check('koha-common Depends: yaz (yaz-marcdump is installed with Koha)', /Depends: yaz$/m.test(u('apt-cache depends koha-common').out)) ctx.check('yaz-marcdump is on the PATH of a normal user', u('command -v yaz-marcdump').out === '/usr/bin/yaz-marcdump') put('src.xml', `${XML}\n`) u('yaz-marcdump -i marcxml -o marc -f UTF-8 -t MARC-8 -l 9=32 src.xml > records.mrc') const head8 = u('yaz-marcdump records.mrc | head -n 3').out ctx.check('MARC-8 file: head -n 3 → "00155nam 2200061 a 4500" / "001 kb40-1" / "100 1 $a Garc�ia M�arquez, Gabriel."', head8 === '00155nam 2200061 a 4500\n001 kb40-1\n100 1 $a Garc�ia M�arquez, Gabriel.', JSON.stringify(head8)) const conv = u('yaz-marcdump -f MARC-8 -t UTF-8 -o marc -l 9=97 records.mrc > records-utf8.mrc') ctx.check('the MARC-8 → UTF-8 conversion prints nothing, exit 0', conv.out === '' && conv.code === 0, JSON.stringify(conv)) const headU = u('yaz-marcdump records-utf8.mrc | head -n 3').out ctx.check('converted: "00160nam a2200061 a 4500" / "001 kb40-1" / "100 1 $a García Márquez, Gabriel." (as displayed)', headU.normalize('NFC') === '00160nam a2200061 a 4500\n001 kb40-1\n100 1 $a García Márquez, Gabriel.', JSON.stringify(headU)) ctx.check('…and the 245 reads "Cien años de soledad / $c Gabriel García Márquez."', u('yaz-marcdump records-utf8.mrc').out.normalize('NFC').includes('245 10 $a Cien años de soledad / $c Gabriel García Márquez.')) ctx.check('yaz writes the accented letters decomposed: "í" is "i" + U+0301 (combining acute), so it looks the same', headU.includes('Garci\u0301a') && !headU.includes('Garc\u00eda'), JSON.stringify(headU.slice(-30))) put('mk.py', `${PY}\n`) ctx.check('Windows-1256 has no U+06CC (Urdu ی)', u('python3 mk.py').out === '06cc-not-in-cp1256') const raw = u('yaz-marcdump urdu.mrc').out ctx.check('CP1256 file as UTF-8: "245 00 $a ����� ����"', raw.includes('245 00 $a ����� ����'), JSON.stringify(raw)) const c2 = u('yaz-marcdump -f CP1256 -t UTF-8 -o marc -l 9=97 urdu.mrc > urdu-utf8.mrc') ctx.check('-f CP1256 is accepted, prints nothing', c2.out === '' && c2.code === 0, JSON.stringify(c2)) const dumpU = u('yaz-marcdump urdu-utf8.mrc').out ctx.check('converted Urdu record: "00081nam a2200049 a 4500" / "001 kb40-2" / "245 00 $a ديوان غالب"', dumpU === '00081nam a2200049 a 4500\n001 kb40-2\n245 00 $a ديوان غالب', JSON.stringify(dumpU)) const xml = u('yaz-marcdump -o marcxml urdu-utf8.mrc').out ctx.check('…and holds the Arabic yeh U+064A, not U+06CC', xml.includes('ي') && !xml.includes('ی')) L('rm -f src.xml records.mrc records-utf8.mrc mk.py urdu.mrc urdu-utf8.mrc') const env = L(`grep PRETTY_NAME /etc/os-release; perl -I/usr/share/koha/lib -MKoha -e 'print qq{KOHA=}, Koha::version, qq{\\n}'; sudo koha-mysql library -N -B -e "SELECT CONCAT('DB=', value) FROM systempreferences WHERE variable='Version'"`) ctx.ranIn({ osRelease: (env.match(/^PRETTY_NAME=.*$/m) ?? [''])[0], kohaVersion: (env.match(/^KOHA=(\S+)/m) ?? [])[1] ?? '', db: (env.match(/^DB=(\S+)/m) ?? [])[1] ?? '' }, 'A fresh Ubuntu 24.04 server with Koha installed from the Koha repository as our installation guide leaves it (yaz comes with koha-common), reached over SSH: every command run as the sudo user librarian on a terminal, in its home folder.') }