/// Generates lib/data/ducet_order.dart from Unicode allkeys.txt and CLDR FractionalUCA.txt. /// /// Usage: /// dart run scripts/generate_ducet_ranks.dart [allkeys.txt] [FractionalUCA.txt] /// /// Downloads the files automatically if not provided. library; import 'dart:io'; // --------------------------------------------------------------------------- // Weight tuple for multi-level UCA comparison // --------------------------------------------------------------------------- class Weight implements Comparable { final List<(int, int, int)> levels; final int codepoint; // tiebreaker const Weight(this.levels, this.codepoint); @override int compareTo(Weight other) { final len = levels.length < other.levels.length ? levels.length : other.levels.length; // Primary pass for (var i = 0; i < len; i++) { final cmp = levels[i].$1.compareTo(other.levels[i].$1); if (cmp != 0) return cmp; } if (levels.length != other.levels.length) { return levels.length.compareTo(other.levels.length); } // Secondary pass for (var i = 0; i < len; i++) { final cmp = levels[i].$2.compareTo(other.levels[i].$2); if (cmp != 0) return cmp; } // Tertiary pass for (var i = 0; i < len; i++) { final cmp = levels[i].$3.compareTo(other.levels[i].$3); if (cmp != 0) return cmp; } // Codepoint tiebreaker return codepoint.compareTo(other.codepoint); } } // --------------------------------------------------------------------------- // Katakana codepoint ranges (for CLDR kana tertiary reversal) // --------------------------------------------------------------------------- bool _isKatakana(int cp) => (cp >= 0x30A0 && cp <= 0x30FF) || // Katakana (cp >= 0x31F0 && cp <= 0x31FF) || // Katakana Phonetic Extensions (cp >= 0xFF65 && cp <= 0xFF9F); // Halfwidth Katakana // --------------------------------------------------------------------------- // Parse allkeys.txt → Map // // CLDR kana fix: ICU/CLDR root collation sorts katakana before hiragana // (opposite of raw DUCET). We adjust by subtracting 6 from katakana tertiary // weights, placing them below hiragana tertiaries (0x000D+). // --------------------------------------------------------------------------- Map parseAllKeys(String text) { final result = {}; final weightRe = RegExp(r'\[([.*])([0-9A-Fa-f]{4})\.([0-9A-Fa-f]{4})\.([0-9A-Fa-f]{4})\]'); for (final line in text.split('\n')) { final trimmed = line.trim(); if (trimmed.isEmpty || trimmed.startsWith('#') || trimmed.startsWith('@')) continue; final semiIdx = trimmed.indexOf(';'); if (semiIdx < 0) continue; final cpPart = trimmed.substring(0, semiIdx).trim(); if (cpPart.contains(' ')) continue; // skip multi-codepoint final cp = int.tryParse(cpPart, radix: 16); if (cp == null || cp > 0xFFFF) continue; // BMP only final weightPart = trimmed.substring(semiIdx + 1); final matches = weightRe.allMatches(weightPart); if (matches.isEmpty) continue; final isKat = _isKatakana(cp); final levels = <(int, int, int)>[]; for (final m in matches) { final p = int.parse(m.group(2)!, radix: 16); final s = int.parse(m.group(3)!, radix: 16); var t = int.parse(m.group(4)!, radix: 16); // CLDR kana fix: lower katakana tertiary below hiragana range if (isKat && t >= 0x000F) t -= 6; levels.add((p, s, t)); } if (levels.every((l) => l.$1 == 0 && l.$2 == 0 && l.$3 == 0)) continue; result[cp] = Weight(levels, cp); } return result; } // --------------------------------------------------------------------------- // Parse FractionalUCA.txt radical lines → CJK codepoints in radical-stroke // order, plus Kangxi radical → CJK decomposition map. // --------------------------------------------------------------------------- (List, Map) parseRadicals(String text) { final order = []; final kangxiDecomp = {}; // kangxi radical CP → CJK CP final radicalRe = RegExp(r'^\[radical \d+'); for (final line in text.split('\n')) { if (!radicalRe.hasMatch(line)) continue; // Parse header: [radical N=:] final eqIdx = line.indexOf('='); final colonIdx = line.indexOf(':'); if (eqIdx < 0 || colonIdx < 0 || colonIdx <= eqIdx) continue; final closeBracket = line.lastIndexOf(']'); if (closeBracket < 0) continue; // Extract Kangxi radical → CJK mapping from header final headerChars = line.substring(eqIdx + 1, colonIdx).runes.toList(); if (headerChars.length >= 2) { final kangxi = headerChars.first; final cjk = headerChars[1]; // Kangxi Radicals: U+2F00-U+2FD5 if (kangxi >= 0x2F00 && kangxi <= 0x2FD5 && cjk <= 0xFFFF) { kangxiDecomp[kangxi] = cjk; } } // Parse character list after colon final charList = line.substring(colonIdx + 1, closeBracket); final runes = charList.runes.toList(); var i = 0; while (i < runes.length) { final cp = runes[i]; if (cp == 0x20) { i++; continue; } // Check for range: - if (i + 2 < runes.length && runes[i + 1] == 0x2D) { final endCp = runes[i + 2]; for (var c = cp; c <= endCp; c++) { if (c <= 0xFFFF) order.add(c); } i += 3; continue; } if (cp <= 0xFFFF) order.add(cp); i++; } } // Deduplicate preserving order final seen = {}; final deduped = order.where((cp) => seen.add(cp)).toList(); return (deduped, kangxiDecomp); } // --------------------------------------------------------------------------- // Build final ordered list // --------------------------------------------------------------------------- List buildOrder(Map allKeys, List cjkRadicalOrder) { final entries = allKeys.entries.toList(); entries.sort((a, b) => a.value.compareTo(b.value)); final ordered = []; final cjkSet = cjkRadicalOrder.toSet(); bool cjkInserted = false; for (final e in entries) { if (cjkSet.contains(e.key)) continue; if (!cjkInserted && e.value.levels.isNotEmpty && e.value.levels.first.$1 >= 0xFB00) { for (final cp in cjkRadicalOrder) { if (cp <= 0xFFFF) ordered.add(cp); } cjkInserted = true; } ordered.add(e.key); } if (!cjkInserted) { for (final cp in cjkRadicalOrder) { if (cp <= 0xFFFF) ordered.add(cp); } } return ordered; } // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- Future main(List args) async { final allKeysPath = args.isNotEmpty ? args.first : '/tmp/allkeys.txt'; final fracUcaPath = args.length > 1 ? args[1] : '/tmp/FractionalUCA.txt'; // Download if missing if (!File(allKeysPath).existsSync()) { stderr.writeln('Downloading allkeys.txt...'); final result = await Process.run('curl', [ '-sL', 'https://www.unicode.org/Public/UCA/13.0.0/allkeys.txt', '-o', allKeysPath, ]); if (result.exitCode != 0) { stderr.writeln('Failed to download allkeys.txt'); exit(1); } } if (!File(fracUcaPath).existsSync()) { stderr.writeln('Downloading FractionalUCA.txt...'); final result = await Process.run('curl', [ '-sL', 'https://raw.githubusercontent.com/unicode-org/cldr/release-39/common/uca/FractionalUCA.txt', '-o', fracUcaPath, ]); if (result.exitCode != 0) { stderr.writeln('Failed to download FractionalUCA.txt'); exit(1); } } stderr.writeln('Parsing allkeys.txt (with CLDR kana fix)...'); final allKeysText = File(allKeysPath).readAsStringSync(); final allKeys = parseAllKeys(allKeysText); stderr.writeln(' ${allKeys.length} BMP entries'); stderr.writeln('Parsing FractionalUCA.txt...'); final fracText = File(fracUcaPath).readAsStringSync(); final (cjkOrder, kangxiDecomp) = parseRadicals(fracText); stderr.writeln(' ${cjkOrder.length} CJK codepoints in radical-stroke order'); stderr.writeln(' ${kangxiDecomp.length} Kangxi radical decompositions'); final bmpCjk = cjkOrder.where((cp) => cp <= 0xFFFF).length; stderr.writeln(' $bmpCjk BMP CJK codepoints'); stderr.writeln('Building total order...'); final ordered = buildOrder(allKeys, cjkOrder); stderr.writeln(' ${ordered.length} total BMP codepoints in order'); stderr.writeln(' ${ordered.length * 2} raw bytes'); // Generate Dart file — store codepoints as a raw UTF-16 string constant. // Each char IS the codepoint. Dart strings are UTF-16 natively, so the // compiled snapshot stores the data as raw 2-byte values with zero decoding. final outPath = '${Directory.current.path}/lib/data/ducet_order.dart'; final buf = StringBuffer(); buf.writeln('/// Sorted BMP codepoints per DUCET (Unicode 13.0) + CLDR 39 CJK radical-stroke.'); buf.writeln('/// Katakana sorts before hiragana (CLDR root tailoring).'); buf.writeln('/// Generated by scripts/generate_ducet_ranks.dart — do not edit.'); buf.writeln(); // Raw UTF-16 string — each code unit is a codepoint in DUCET order. // Use \uXXXX escapes so the source stays ASCII-safe. buf.write("const String _ducetOrder = '"); for (var i = 0; i < ordered.length; i++) { final cp = ordered[i]; if (cp == 0x27) { buf.write(r"\'"); // escape single quote } else if (cp == 0x5C) { buf.write(r'\\'); // escape backslash } else if (cp == 0x24) { buf.write(r'\$'); // escape dollar } else { buf.write('\\u${cp.toRadixString(16).padLeft(4, '0')}'); } } buf.writeln("';"); // Kangxi radical decomposition map buf.writeln(); buf.writeln('/// Kangxi Radicals (U+2F00-U+2FD5) → CJK Unified equivalents.'); buf.writeln('/// ICU decomposes these via NFD before collation.'); buf.writeln('const Map _kangxiDecomp = {'); final sortedKangxi = kangxiDecomp.entries.toList()..sort((a, b) => a.key.compareTo(b.key)); for (final e in sortedKangxi) { buf.writeln(' 0x${e.key.toRadixString(16).toUpperCase()}: 0x${e.value.toRadixString(16).toUpperCase()},'); } buf.writeln('};'); buf.writeln(); buf.writeln('late final Map _ranks = _buildRanks();'); buf.writeln(); buf.writeln('Map _buildRanks() {'); buf.writeln(' return {'); buf.writeln(' for (var i = 0; i < _ducetOrder.length; i++)'); buf.writeln(' _ducetOrder.codeUnitAt(i): i,'); buf.writeln(' };'); buf.writeln('}'); buf.writeln(); buf.writeln('/// Compare two characters using DUCET + CLDR ordering.'); buf.writeln('/// Decomposes Kangxi radicals to CJK equivalents before lookup.'); buf.writeln('/// Falls back to codepoint order for characters not in the table.'); buf.writeln('int ducetCompare(String a, String b) {'); buf.writeln(' var cpA = a.runes.first;'); buf.writeln(' var cpB = b.runes.first;'); buf.writeln(' cpA = _kangxiDecomp[cpA] ?? cpA;'); buf.writeln(' cpB = _kangxiDecomp[cpB] ?? cpB;'); buf.writeln(' final rankA = _ranks[cpA] ?? (0x100000 + cpA);'); buf.writeln(' final rankB = _ranks[cpB] ?? (0x100000 + cpB);'); buf.writeln(' return rankA.compareTo(rankB);'); buf.writeln('}'); File(outPath).writeAsStringSync(buf.toString()); stderr.writeln('Written to $outPath'); }