Koha Solutions knowledge base: run log of a live check Article https://kohasolutions.com/knowledge-base/koha-broken-characters-after-import Check scripts/kb/live/koha-broken-characters-after-import.mjs (SHA-256 6838a9509618e612ac7db4ad55a5f4be5489fb6b51bda1d279bf3097c5f9a3e6) Ran 2026-11-05T14:46:53Z (UTC), 3 s Koha 26.05.05 (Koha::version 26.05.05.000, database version none (no Koha database installed in the box)) Source https://git.koha-community.org/Koha-community/Koha/src/commit/d3fc229bc5eaea25dfe88f86149386408fe5d32d System Ubuntu 24.04.5 LTS Setup A clean Ubuntu 24.04 container with koha-common 26.05.05 (and its yaz dependency) from the Koha repository; no instance needed. Result PASS: all 11 claims held Each claim is something the article says. "seen" is what the check read back when it is not just a yes/no. Changes the check made to the test server were undone afterwards, as its header describes. == Claims == ok koha-common Depends: yaz (yaz-marcdump is installed with Koha) ok MARC-8 file: head -n 3 → "00155nam 2200061 a 4500" / "001 kb40-1" / "100 1 $a Garc�ia M�arquez, Gabriel." seen: "00155nam 2200061 a 4500\n001 kb40-1\n100 1 $a Garc�ia M�arquez, Gabriel." ok the MARC-8 → UTF-8 conversion prints nothing, exit 0 seen: {"out":"","code":0} ok converted: "00160nam a2200061 a 4500" / "001 kb40-1" / "100 1 $a García Márquez, Gabriel." (as displayed) seen: "00160nam a2200061 a 4500\n001 kb40-1\n100 1 $a García Márquez, Gabriel." ok …and the 245 reads "Cien años de soledad / $c Gabriel García Márquez." ok yaz writes the accented letters decomposed: "í" is "i" + U+0301 (combining acute), so it looks the same seen: " $a García Márquez, Gabriel." ok Windows-1256 has no U+06CC (Urdu ی) ok CP1256 file as UTF-8: "245 00 $a ����� ����" seen: "00072nam 2200049 a 4500\n001 kb40-2\n245 00 $a ����� ����" ok -f CP1256 is accepted, prints nothing seen: {"out":"","code":0} ok converted Urdu record: "00081nam a2200049 a 4500" / "001 kb40-2" / "245 00 $a ديوان غالب" seen: "00081nam a2200049 a 4500\n001 kb40-2\n245 00 $a ديوان غالب" ok …and holds the Arabic yeh U+064A, not U+06CC == The check that produced this log (scripts/kb/live/koha-broken-characters-after-import.mjs) == /** * Live check for prisma/ks-kb/40-koha-broken-characters-after-import.ts, the yaz-marcdump blocks. Does NOT touch live * Koha. Runs in a throwaway `kbv-koha-ubuntu:24.04` container (koha-common 26.05.05, which Depends on yaz): makes a * MARC-8 file from MARCXML with yaz-marcdump itself, and a Windows-1256 Urdu record byte by byte (yaz cannot write * CP1256), then runs the article's commands and asserts the responses it shows. The staging behaviour (�, the log-only * warning) is from the 2026-11-04 run on live Koha recorded in the article's header. About 20 seconds. */ import { spawnSync } from 'node:child_process' export const meta = { koha: '26.05.05', passed: '2026-11-05' } const IMG = 'kbv-koha-ubuntu:24.04' const BOX = 'kbv-40-check' const XML = ` 00000nam a2200000 a 4500 kb40-1 García Márquez, Gabriel. Cien años de soledad /Gabriel García Márquez. ` const PY = `FT, SF, RT = b"\\x1e", b"\\x1f", b"\\x1d" def rec(fields): data, dirs, pos = b"", b"", 0 for tag, body in fields: b = body + FT dirs += tag.encode() + b"%04d%05d" % (len(b), pos); pos += len(b); data += b base = 24 + len(dirs) + 1 return (b"%05dnam 22%05d a 4500" % (base + len(data) + 1, base)) + dirs + FT + data + RT open("urdu.mrc", "wb").write(rec([("001", b"kb40-2"), ("245", b"00" + SF + b"a" + "\\u062f\\u064a\\u0648\\u0627\\u0646 \\u063a\\u0627\\u0644\\u0628".encode("cp1256"))])) try: "\\u06cc".encode("cp1256"); print("06cc-in-cp1256") except UnicodeEncodeError: print("06cc-not-in-cp1256")` export default async function run(ctx) { if (spawnSync('docker', ['image', 'inspect', IMG]).status !== 0) { ctx.check(`image ${IMG} present`, false) return } spawnSync('docker', ['rm', '-f', BOX]) ctx.restore('remove the container', () => spawnSync('docker', ['rm', '-f', BOX])) spawnSync('docker', ['run', '-d', '--name', BOX, IMG, 'sleep', 'infinity']) const u = (cmd) => { const r = spawnSync('docker', ['exec', '-w', '/tmp', BOX, 'bash', '-c', `{ ${cmd} } 2>&1`], { encoding: 'utf8', timeout: 120000 }) return { out: (r.stdout ?? '').replace(/\s+$/, ''), code: r.status ?? -1 } } ctx.check('koha-common Depends: yaz (yaz-marcdump is installed with Koha)', /Depends: yaz$/m.test(u('apt-cache depends koha-common').out)) u(`cat > src.xml <<'EOF'\n${XML}\nEOF\nyaz-marcdump -i marcxml -o marc -f UTF-8 -t MARC-8 -l 9=32 src.xml > records.mrc`) const head8 = u('yaz-marcdump records.mrc | head -n 3').out ctx.check('MARC-8 file: head -n 3 → "00155nam 2200061 a 4500" / "001 kb40-1" / "100 1 $a Garc�ia M�arquez, Gabriel."', head8 === '00155nam 2200061 a 4500\n001 kb40-1\n100 1 $a Garc�ia M�arquez, Gabriel.', JSON.stringify(head8)) const conv = u('yaz-marcdump -f MARC-8 -t UTF-8 -o marc -l 9=97 records.mrc > records-utf8.mrc') ctx.check('the MARC-8 → UTF-8 conversion prints nothing, exit 0', conv.out === '' && conv.code === 0, JSON.stringify(conv)) const headU = u('yaz-marcdump records-utf8.mrc | head -n 3').out ctx.check('converted: "00160nam a2200061 a 4500" / "001 kb40-1" / "100 1 $a García Márquez, Gabriel." (as displayed)', headU.normalize('NFC') === '00160nam a2200061 a 4500\n001 kb40-1\n100 1 $a García Márquez, Gabriel.', JSON.stringify(headU)) ctx.check('…and the 245 reads "Cien años de soledad / $c Gabriel García Márquez."', u('yaz-marcdump records-utf8.mrc').out.normalize('NFC').includes('245 10 $a Cien años de soledad / $c Gabriel García Márquez.')) ctx.check('yaz writes the accented letters decomposed: "í" is "i" + U+0301 (combining acute), so it looks the same', headU.includes('Garci\u0301a') && !headU.includes('Garc\u00eda'), JSON.stringify(headU.slice(-30))) u(`cat > mk.py <<'EOF'\n${PY}\nEOF`) ctx.check('Windows-1256 has no U+06CC (Urdu ی)', u('python3 mk.py').out === '06cc-not-in-cp1256') const raw = u('yaz-marcdump urdu.mrc').out ctx.check('CP1256 file as UTF-8: "245 00 $a ����� ����"', raw.includes('245 00 $a ����� ����'), JSON.stringify(raw)) const c2 = u('yaz-marcdump -f CP1256 -t UTF-8 -o marc -l 9=97 urdu.mrc > urdu-utf8.mrc') ctx.check('-f CP1256 is accepted, prints nothing', c2.out === '' && c2.code === 0, JSON.stringify(c2)) const dumpU = u('yaz-marcdump urdu-utf8.mrc').out ctx.check('converted Urdu record: "00081nam a2200049 a 4500" / "001 kb40-2" / "245 00 $a ديوان غالب"', dumpU === '00081nam a2200049 a 4500\n001 kb40-2\n245 00 $a ديوان غالب', JSON.stringify(dumpU)) ctx.check('…and holds the Arabic yeh U+064A, not U+06CC', u('yaz-marcdump -o marcxml urdu-utf8.mrc').out.includes('ي') && !u('yaz-marcdump -o marcxml urdu-utf8.mrc').out.includes('ی')) ctx.ranInBox(BOX, 'A clean Ubuntu 24.04 container with koha-common 26.05.05 (and its yaz dependency) from the Koha repository; no instance needed.') }