355 lines
13 KiB
Python
Executable File
355 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
#
|
|
# Leverage Python's unicodedata module to generate ucs_width_table.h
|
|
|
|
import unicodedata
|
|
import sys
|
|
import argparse
|
|
|
|
# This script's file name
|
|
from pathlib import Path
|
|
this_file = Path(__file__).name
|
|
|
|
# Default output file name
|
|
DEFAULT_OUT_FILE = "ucs_width_table.h"
|
|
|
|
# --- Global Constants for Width Assignments ---
|
|
|
|
# Known zero-width characters
|
|
KNOWN_ZERO_WIDTH = (
|
|
0x200B, # ZERO WIDTH SPACE
|
|
0x200C, # ZERO WIDTH NON-JOINER
|
|
0x200D, # ZERO WIDTH JOINER
|
|
0x2060, # WORD JOINER
|
|
0xFEFF # ZERO WIDTH NO-BREAK SPACE (BOM)
|
|
)
|
|
|
|
# Zero-width emoji modifiers and components
|
|
# NOTE: Some of these characters would normally be single-width according to
|
|
# East Asian Width properties, but we deliberately override them to be
|
|
# zero-width because they function as modifiers in emoji sequences.
|
|
EMOJI_ZERO_WIDTH = [
|
|
# Skin tone modifiers
|
|
(0x1F3FB, 0x1F3FF), # Emoji modifiers (skin tones)
|
|
|
|
# Variation selectors (note: VS16 is treated specially in vt.c)
|
|
(0xFE00, 0xFE0F), # Variation Selectors 1-16
|
|
|
|
# Gender and hair style modifiers
|
|
# These would be single-width by Unicode properties, but are zero-width
|
|
# when part of emoji
|
|
(0x2640, 0x2640), # Female sign
|
|
(0x2642, 0x2642), # Male sign
|
|
(0x26A7, 0x26A7), # Transgender symbol
|
|
(0x1F9B0, 0x1F9B3), # Hair components (red, curly, white, bald)
|
|
|
|
# Tag characters
|
|
(0xE0020, 0xE007E), # Tags
|
|
]
|
|
|
|
# Regional indicators (flag components)
|
|
REGIONAL_INDICATORS = (0x1F1E6, 0x1F1FF) # Regional indicator symbols A-Z
|
|
|
|
# Double-width emoji ranges
|
|
#
|
|
# Many emoji characters are classified as single-width according to Unicode
|
|
# Standard Annex #11 East Asian Width property (N or Neutral), but we
|
|
# deliberately override them to be double-width. References:
|
|
# 1. Unicode Technical Standard #51: Unicode Emoji
|
|
# (https://www.unicode.org/reports/tr51/)
|
|
# 2. Principle of "emoji presentation" in WHATWG CSS Text specification
|
|
# (https://drafts.csswg.org/css-text-3/#character-properties)
|
|
# 3. Terminal emulator implementations (iTerm2, Windows Terminal, etc.) which
|
|
# universally render emoji as double-width characters regardless of their
|
|
# Unicode EAW property
|
|
# 4. W3C Work Item: Requirements for Japanese Text Layout - Section 3.8.1
|
|
# Emoji width (https://www.w3.org/TR/jlreq/)
|
|
EMOJI_RANGES = [
|
|
(0x1F000, 0x1F02F), # Mahjong Tiles (EAW: N, but displayed as double-width)
|
|
(0x1F0A0, 0x1F0FF), # Playing Cards (EAW: N, but displayed as double-width)
|
|
(0x1F300, 0x1F5FF), # Miscellaneous Symbols and Pictographs
|
|
(0x1F600, 0x1F64F), # Emoticons
|
|
(0x1F680, 0x1F6FF), # Transport and Map Symbols
|
|
(0x1F700, 0x1F77F), # Alchemical Symbols
|
|
(0x1F780, 0x1F7FF), # Geometric Shapes Extended
|
|
(0x1F800, 0x1F8FF), # Supplemental Arrows-C
|
|
(0x1F900, 0x1F9FF), # Supplemental Symbols and Pictographs
|
|
(0x1FA00, 0x1FA6F), # Chess Symbols
|
|
(0x1FA70, 0x1FAFF), # Symbols and Pictographs Extended-A
|
|
]
|
|
|
|
def create_width_tables():
|
|
"""
|
|
Creates Unicode character width tables and returns the data structures.
|
|
|
|
Returns:
|
|
tuple: (zero_width_ranges, double_width_ranges)
|
|
"""
|
|
|
|
# Width data mapping
|
|
width_map = {} # Maps code points to width (0, 1, 2)
|
|
|
|
# Mark emoji modifiers as zero-width
|
|
for start, end in EMOJI_ZERO_WIDTH:
|
|
for cp in range(start, end + 1):
|
|
width_map[cp] = 0
|
|
|
|
# Mark all regional indicators as single-width as they are usually paired
|
|
# providing a combined width of 2 when displayed together.
|
|
start, end = REGIONAL_INDICATORS
|
|
for cp in range(start, end + 1):
|
|
width_map[cp] = 1
|
|
|
|
# Process all assigned Unicode code points (Basic Multilingual Plane +
|
|
# Supplementary Planes) Range 0x0 to 0x10FFFF (the full Unicode range)
|
|
for block_start in range(0, 0x110000, 0x1000):
|
|
block_end = block_start + 0x1000
|
|
for cp in range(block_start, block_end):
|
|
try:
|
|
char = chr(cp)
|
|
|
|
# Skip if already processed
|
|
if cp in width_map:
|
|
continue
|
|
|
|
# Check for combining marks and a format characters
|
|
category = unicodedata.category(char)
|
|
|
|
# Combining marks
|
|
if category.startswith('M'):
|
|
width_map[cp] = 0
|
|
continue
|
|
|
|
# Format characters
|
|
# Since we have no support for bidirectional text, all format
|
|
# characters (category Cf) can be treated with width 0 (zero)
|
|
# for simplicity, as they don't need to occupy visual space
|
|
# in a non-bidirectional text environment.
|
|
if category == 'Cf':
|
|
width_map[cp] = 0
|
|
continue
|
|
|
|
# Known zero-width characters
|
|
if cp in KNOWN_ZERO_WIDTH:
|
|
width_map[cp] = 0
|
|
continue
|
|
|
|
# Use East Asian Width property
|
|
eaw = unicodedata.east_asian_width(char)
|
|
if eaw in ('F', 'W'): # Fullwidth or Wide
|
|
width_map[cp] = 2
|
|
elif eaw in ('Na', 'H', 'N', 'A'): # Narrow, Halfwidth, Neutral, Ambiguous
|
|
width_map[cp] = 1
|
|
else:
|
|
# Default to single-width for unknown
|
|
width_map[cp] = 1
|
|
|
|
except (ValueError, OverflowError):
|
|
# Skip invalid code points
|
|
continue
|
|
|
|
# Process Emoji - generally double-width
|
|
for start, end in EMOJI_RANGES:
|
|
for cp in range(start, end + 1):
|
|
if cp not in width_map or width_map[cp] != 0: # Don't override zero-width
|
|
try:
|
|
char = chr(cp)
|
|
width_map[cp] = 2
|
|
except (ValueError, OverflowError):
|
|
continue
|
|
|
|
# Optimize to create range tables
|
|
def ranges_optimize(width_data, target_width):
|
|
points = sorted([cp for cp, width in width_data.items() if width == target_width])
|
|
if not points:
|
|
return []
|
|
|
|
# Group consecutive code points into ranges
|
|
ranges = []
|
|
start = points[0]
|
|
prev = start
|
|
|
|
for cp in points[1:]:
|
|
if cp > prev + 1:
|
|
ranges.append((start, prev))
|
|
start = cp
|
|
prev = cp
|
|
|
|
# Add the last range
|
|
ranges.append((start, prev))
|
|
return ranges
|
|
|
|
# Extract ranges for each width
|
|
zero_width_ranges = ranges_optimize(width_map, 0)
|
|
double_width_ranges = ranges_optimize(width_map, 2)
|
|
|
|
return zero_width_ranges, double_width_ranges
|
|
|
|
def write_tables(zero_width_ranges, double_width_ranges, out_file=DEFAULT_OUT_FILE):
|
|
"""
|
|
Write the generated tables to C header file.
|
|
|
|
The output uses a single sorted-by-`first` table per region (BMP and
|
|
non-BMP), with zero-width and double-width ranges merged together. The
|
|
non-BMP table also hosts the BMP double-width bitmap in spare bits of
|
|
`last`. See the encoding comment at the top of ucs.c for the layout.
|
|
|
|
Args:
|
|
zero_width_ranges: List of (start, end) ranges for zero-width characters
|
|
double_width_ranges: List of (start, end) ranges for double-width characters
|
|
out_file: Output file name (default: DEFAULT_OUT_FILE)
|
|
"""
|
|
|
|
# Bits per BMP-bitmap chunk hosted in one non-BMP entry's `last` field.
|
|
# 8 bits makes `idx / BITS_PER_CHUNK` / `idx % BITS_PER_CHUNK` compile to
|
|
# a cheap shift+mask in the lookup. The chunk size is also emitted as
|
|
# UCS_NONBMP_BMP_BITS in the generated header so ucs.c stays in sync.
|
|
BITS_PER_CHUNK = 8
|
|
|
|
# Function to split ranges into BMP (16-bit) and non-BMP (above 16-bit)
|
|
def split_ranges_by_size(ranges):
|
|
bmp_ranges = []
|
|
non_bmp_ranges = []
|
|
|
|
for start, end in ranges:
|
|
if end <= 0xFFFF:
|
|
bmp_ranges.append((start, end))
|
|
elif start > 0xFFFF:
|
|
non_bmp_ranges.append((start, end))
|
|
else:
|
|
# Split the range at 0xFFFF
|
|
bmp_ranges.append((start, 0xFFFF))
|
|
non_bmp_ranges.append((0x10000, end))
|
|
|
|
return bmp_ranges, non_bmp_ranges
|
|
|
|
# Split ranges into BMP and non-BMP
|
|
zero_width_bmp, zero_width_non_bmp = split_ranges_by_size(zero_width_ranges)
|
|
double_width_bmp, double_width_non_bmp = split_ranges_by_size(double_width_ranges)
|
|
|
|
# Merge zero- and double-width ranges per region, tagging each with its
|
|
# width, then sort by `first` so binary search works on the union.
|
|
bmp_entries = sorted(
|
|
[(s, e, 0) for s, e in zero_width_bmp] +
|
|
[(s, e, 2) for s, e in double_width_bmp],
|
|
key=lambda t: t[0])
|
|
nonbmp_entries = sorted(
|
|
[(s, e, 0) for s, e in zero_width_non_bmp] +
|
|
[(s, e, 2) for s, e in double_width_non_bmp],
|
|
key=lambda t: t[0])
|
|
|
|
# Build the BMP double-width bitmap: one bit per BMP entry (in sort
|
|
# order), set iff that entry is double-width. Pack into BITS_PER_CHUNK-
|
|
# wide chunks, with bit j of the chunk corresponding to entry
|
|
# (chunk_index * BITS_PER_CHUNK + j).
|
|
bmp_w2_bits = [1 if w == 2 else 0 for _, _, w in bmp_entries]
|
|
n_chunks = (len(bmp_w2_bits) + BITS_PER_CHUNK - 1) // BITS_PER_CHUNK
|
|
|
|
if n_chunks > len(nonbmp_entries):
|
|
raise RuntimeError(
|
|
f"BMP bitmap needs {n_chunks} host entries, "
|
|
f"but only {len(nonbmp_entries)} non-BMP entries are available")
|
|
|
|
chunks = [] # list of (base_index, end_index, packed_value)
|
|
for c in range(n_chunks):
|
|
base = c * BITS_PER_CHUNK
|
|
end_idx = min(base + BITS_PER_CHUNK - 1, len(bmp_w2_bits) - 1)
|
|
value = 0
|
|
for j in range(BITS_PER_CHUNK):
|
|
k = base + j
|
|
if k < len(bmp_w2_bits) and bmp_w2_bits[k]:
|
|
value |= 1 << j
|
|
chunks.append((base, end_idx, value))
|
|
|
|
# Function to generate code point description comments
|
|
def get_code_point_comment(start, end):
|
|
try:
|
|
start_char_desc = unicodedata.name(chr(start))
|
|
if start == end:
|
|
return f"/* {start_char_desc} */"
|
|
else:
|
|
end_char_desc = unicodedata.name(chr(end))
|
|
return f"/* {start_char_desc} - {end_char_desc} */"
|
|
except:
|
|
if start == end:
|
|
return f"/* U+{start:04X} */"
|
|
else:
|
|
return f"/* U+{start:04X} - U+{end:04X} */"
|
|
|
|
# Generate C tables
|
|
with open(out_file, 'w') as f:
|
|
f.write(f"""\
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* {out_file} - Unicode character width
|
|
*
|
|
* Auto-generated by {this_file}
|
|
*
|
|
* Unicode Version: {unicodedata.unidata_version}
|
|
*
|
|
* Zero-width and double-width ranges are merged into one sorted-by-`first`
|
|
* table per region. The non-BMP table additionally hosts the BMP
|
|
* double-width bitmap in the low {BITS_PER_CHUNK} bits of `last` of its
|
|
* first {n_chunks} entries (covering {len(bmp_w2_bits)} BMP entries).
|
|
* See ucs.c for the encoding details and the lookup code.
|
|
*/
|
|
|
|
/* Bits per BMP-bitmap chunk hosted in one non-BMP entry's `last` field. */
|
|
#define UCS_NONBMP_BMP_BITS {BITS_PER_CHUNK}
|
|
|
|
/* Combined zero- and double-width ranges
|
|
* (BMP - Basic Multilingual Plane, U+0000 to U+FFFF). */
|
|
static const struct ucs_width16 ucs_bmp_ranges[] = {{
|
|
""")
|
|
|
|
for s, e, w in bmp_entries:
|
|
macro = "BMP_0WIDTH" if w == 0 else "BMP_2WIDTH"
|
|
comment = get_code_point_comment(s, e)
|
|
f.write(f"\t{{ {macro}(0x{s:04X}, 0x{e:04X}) }}, {comment}\n")
|
|
|
|
f.write(f"""\
|
|
}};
|
|
|
|
/* Combined zero- and double-width ranges (non-BMP, U+10000 and above).
|
|
* The first {n_chunks} entries host the BMP double-width bitmap in the low
|
|
* {BITS_PER_CHUNK} bits of `last`. */
|
|
static const struct ucs_width32 ucs_nonbmp_ranges[] = {{
|
|
""")
|
|
|
|
for i, (s, e, w) in enumerate(nonbmp_entries):
|
|
macro = "RANGE_0WIDTH" if w == 0 else "RANGE_2WIDTH"
|
|
comment = get_code_point_comment(s, e)
|
|
if i < len(chunks):
|
|
base, end_idx, value = chunks[i]
|
|
f.write(
|
|
f"\t{{ {macro}(0x{s:05X}, 0x{e:05X}) {comment}\n"
|
|
f"\t | BMP_2W_BITS(0b{value:0{BITS_PER_CHUNK}b}) }},"
|
|
f" /* BMP entries [{base:>3}..{end_idx:>3}] */\n")
|
|
else:
|
|
f.write(f"\t{{ {macro}(0x{s:05X}, 0x{e:05X}) }}, {comment}\n")
|
|
|
|
f.write("};\n")
|
|
|
|
if __name__ == "__main__":
|
|
# Parse command line arguments
|
|
parser = argparse.ArgumentParser(description="Generate Unicode width tables")
|
|
parser.add_argument("-o", "--output", dest="output_file", default=DEFAULT_OUT_FILE,
|
|
help=f"Output file name (default: {DEFAULT_OUT_FILE})")
|
|
args = parser.parse_args()
|
|
|
|
# Write tables to header file
|
|
zero_width_ranges, double_width_ranges = create_width_tables()
|
|
write_tables(zero_width_ranges, double_width_ranges, out_file=args.output_file)
|
|
|
|
# Print summary
|
|
zero_width_count = sum(end - start + 1 for start, end in zero_width_ranges)
|
|
double_width_count = sum(end - start + 1 for start, end in double_width_ranges)
|
|
n_zero = len(zero_width_ranges)
|
|
n_double = len(double_width_ranges)
|
|
print(f"Generated {args.output_file} with:")
|
|
print(f"- {n_zero} zero-width ranges covering ~{zero_width_count} code points")
|
|
print(f"- {n_double} double-width ranges covering ~{double_width_count} code points")
|
|
print(f"- {n_zero + n_double} merged ranges total")
|
|
print(f"- Unicode Version: {unicodedata.unidata_version}")
|