From 96656614454f96034d9cc0f0e14c76493b5b7c6b Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Mon, 24 Jun 2024 21:18:37 +0200 Subject: vt: only apply VS-15/16 to valid sequences At compile time, build a lookup table from the Unicode data file 'emoji-variation-sequences.txt'. At run-time, when we detect a VS-15/16 sequence, do a lookup in this table, and enforce the variation selector iff the sequence is valid. Closes #1742 --- scripts/generate-emoji-variation-sequences.py | 95 +++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 scripts/generate-emoji-variation-sequences.py (limited to 'scripts') diff --git a/scripts/generate-emoji-variation-sequences.py b/scripts/generate-emoji-variation-sequences.py new file mode 100644 index 00000000..1a4069ac --- /dev/null +++ b/scripts/generate-emoji-variation-sequences.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import argparse +import sys + + +class Codepoint: + def __init__(self, start: int, end: None|int = None): + self.start = start + self.end = start if end is None else end + self.vs15 = False + self.vs16 = False + + def __repr__(self) -> str: + return f'{self.start:x}-{self.end:x}, vs15={self.vs15}, vs16={self.vs16}' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input', type=argparse.FileType('r')) + parser.add_argument('output', type=argparse.FileType('w')) + opts = parser.parse_args() + + codepoints: list[Codepoint] = [] + + for line in opts.input: + line = line.rstrip() + if not line: + continue + if line[0] == '#': + continue + + cp, vs, _ = line.split(' ', maxsplit=2) + cp = int(cp, 16) + vs = int(vs, 16) + + assert vs == 0xfe0e or vs == 0xfe0f + + if len(codepoints) == 0 or codepoints[-1].start != cp: + codepoints.append(Codepoint(cp)) + else: + assert codepoints[-1].start == cp + + if vs == 0xfe0e: + codepoints[-1].vs15 = True + else: + codepoints[-1].vs16 = True + + + compacted_codepoints: list[Codepoint] = [] + for i, cp in enumerate(codepoints): + assert cp.end == cp.start + + if i == 0: + compacted_codepoints.append(cp) + continue + + last_cp = compacted_codepoints[-1] + if last_cp.end == cp.start - 1 and last_cp.vs15 == cp.vs15 and last_cp.vs16 == cp.vs16: + compacted_codepoints[-1].end = cp.start + else: + compacted_codepoints.append(cp) + + opts.output.write('#pragma once\n') + opts.output.write('#include \n') + opts.output.write('#include \n') + opts.output.write('\n') + opts.output.write('struct emoji_vs {\n') + opts.output.write(' uint32_t start;\n') + opts.output.write(' uint32_t end;\n') + opts.output.write(' bool vs15:1;\n') + opts.output.write(' bool vs16:1;\n') + opts.output.write('} __attribute__((packed));\n') + opts.output.write('_Static_assert(sizeof(struct emoji_vs) == 9, "unexpected struct size");\n') + opts.output.write('\n') + opts.output.write('#if defined(FOOT_GRAPHEME_CLUSTERING)\n') + opts.output.write('\n') + + opts.output.write(f'static const struct emoji_vs emoji_vs[{len(compacted_codepoints)}] = {{\n') + + for cp in compacted_codepoints: + opts.output.write(' {\n') + opts.output.write(f' .start = 0x{cp.start:X},\n') + opts.output.write(f' .end = 0x{cp.end:x},\n') + opts.output.write(f' .vs15 = {"true" if cp.vs15 else "false"},\n') + opts.output.write(f' .vs16 = {"true" if cp.vs16 else "false"},\n') + opts.output.write(' },\n') + + opts.output.write('};\n') + opts.output.write('\n') + opts.output.write('#endif /* FOOT_GRAPHEME_CLUSTERING */\n') + + +if __name__ == '__main__': + sys.exit(main()) -- cgit v1.3 From ecb1ca61afe3c3b3ff676bfc815996b5b2ffd26d Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Tue, 25 Jun 2024 08:23:27 +0200 Subject: scripts: generate-emoji-variation-sequences: don't assume input is sorted --- scripts/generate-emoji-variation-sequences.py | 31 ++++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'scripts') diff --git a/scripts/generate-emoji-variation-sequences.py b/scripts/generate-emoji-variation-sequences.py index 1a4069ac..7324ae3b 100644 --- a/scripts/generate-emoji-variation-sequences.py +++ b/scripts/generate-emoji-variation-sequences.py @@ -21,7 +21,7 @@ def main(): parser.add_argument('output', type=argparse.FileType('w')) opts = parser.parse_args() - codepoints: list[Codepoint] = [] + codepoints: dict[int, Codepoint] = {} for line in opts.input: line = line.rstrip() @@ -36,30 +36,31 @@ def main(): assert vs == 0xfe0e or vs == 0xfe0f - if len(codepoints) == 0 or codepoints[-1].start != cp: - codepoints.append(Codepoint(cp)) - else: - assert codepoints[-1].start == cp + if cp not in codepoints: + codepoints[cp] = Codepoint(cp) + + assert codepoints[cp].start == cp if vs == 0xfe0e: - codepoints[-1].vs15 = True + codepoints[cp].vs15 = True else: - codepoints[-1].vs16 = True + codepoints[cp].vs16 = True + sorted_list = sorted(codepoints.values(), key=lambda cp: cp.start) - compacted_codepoints: list[Codepoint] = [] - for i, cp in enumerate(codepoints): + compacted: list[Codepoint] = [] + for i, cp in enumerate(sorted_list): assert cp.end == cp.start if i == 0: - compacted_codepoints.append(cp) + compacted.append(cp) continue - last_cp = compacted_codepoints[-1] + last_cp = compacted[-1] if last_cp.end == cp.start - 1 and last_cp.vs15 == cp.vs15 and last_cp.vs16 == cp.vs16: - compacted_codepoints[-1].end = cp.start + compacted[-1].end = cp.start else: - compacted_codepoints.append(cp) + compacted.append(cp) opts.output.write('#pragma once\n') opts.output.write('#include \n') @@ -76,9 +77,9 @@ def main(): opts.output.write('#if defined(FOOT_GRAPHEME_CLUSTERING)\n') opts.output.write('\n') - opts.output.write(f'static const struct emoji_vs emoji_vs[{len(compacted_codepoints)}] = {{\n') + opts.output.write(f'static const struct emoji_vs emoji_vs[{len(compacted)}] = {{\n') - for cp in compacted_codepoints: + for cp in compacted: opts.output.write(' {\n') opts.output.write(f' .start = 0x{cp.start:X},\n') opts.output.write(f' .end = 0x{cp.end:x},\n') -- cgit v1.3 From aed9c392ebdf1b030f819ad3e55d21d80dbee2cf Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Tue, 25 Jun 2024 16:22:22 +0200 Subject: scripts: generate-emoji-variation-sequences: compact the C struct Lookups in this table is not performance critical at all. Thus, let's compact it to bring down the binary size of foot. This brings the size of each entry down from 9 bytes to 6, bringing the size of the whole thing down from 1647 bytes to 1098 bytes, saving us 549 bytes. --- scripts/generate-emoji-variation-sequences.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/generate-emoji-variation-sequences.py b/scripts/generate-emoji-variation-sequences.py index 7324ae3b..2c71594c 100644 --- a/scripts/generate-emoji-variation-sequences.py +++ b/scripts/generate-emoji-variation-sequences.py @@ -67,12 +67,12 @@ def main(): opts.output.write('#include \n') opts.output.write('\n') opts.output.write('struct emoji_vs {\n') - opts.output.write(' uint32_t start;\n') - opts.output.write(' uint32_t end;\n') + opts.output.write(' uint32_t start:21;\n') + opts.output.write(' uint32_t end:21;\n') opts.output.write(' bool vs15:1;\n') opts.output.write(' bool vs16:1;\n') opts.output.write('} __attribute__((packed));\n') - opts.output.write('_Static_assert(sizeof(struct emoji_vs) == 9, "unexpected struct size");\n') + opts.output.write('_Static_assert(sizeof(struct emoji_vs) == 6, "unexpected struct size");\n') opts.output.write('\n') opts.output.write('#if defined(FOOT_GRAPHEME_CLUSTERING)\n') opts.output.write('\n') -- cgit v1.3 From 085c60a33423bf46f64f225e438f62ad12fff8c9 Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Wed, 26 Jun 2024 18:30:17 +0200 Subject: scripts: generate-emoji-variation-sequences: don't assume single codepoint sequences Right now (Unicode 15.1), all valid variation sequences consist of a single Unicode codepoint (followed by either VS-15 or VS-16). Don't assume this is the case. We don't actually handle longer sequences. But now we at least catch such escapes, and error out. --- scripts/generate-emoji-variation-sequences.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/generate-emoji-variation-sequences.py b/scripts/generate-emoji-variation-sequences.py index 2c71594c..e05b6290 100644 --- a/scripts/generate-emoji-variation-sequences.py +++ b/scripts/generate-emoji-variation-sequences.py @@ -30,11 +30,18 @@ def main(): if line[0] == '#': continue - cp, vs, _ = line.split(' ', maxsplit=2) - cp = int(cp, 16) - vs = int(vs, 16) + # Example: "0023 FE0E ; text style; # (1.1) NUMBER SIGN" + cps, _ = line.split(';', maxsplit=1) # cps = "0023 FE0F " + cps = cps.strip().split(' ') # cps = ["0023", "FE0F"] - assert vs == 0xfe0e or vs == 0xfe0f + if len(cps) != 2: + raise NotImplementedError(f'emoji variation sequences with more than one base codepoint: {cps}') + + cp, vs = cps # cp = "0023", vs = "FE0F" + cp = int(cp, 16) # cp = 0x23 + vs = int(vs, 16) # vs = 0xfe0f + + assert vs in [0xfe0e, 0xfe0f] if cp not in codepoints: codepoints[cp] = Codepoint(cp) -- cgit v1.3