From 4283a8c51b37330ded50e49f61c6c9a3a4fe19b8 Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Mon, 27 Apr 2020 12:13:30 +0200 Subject: utf8: add support for unicode combining characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This feature lets foot combine e.g. "a\u0301" to "á". We first check if the current character (that we're about to print) is a combining character, by checking if it's in one of the following ranges: * Combining Diacritical Marks (0300–036F), since version 1.0, with modifications in subsequent versions down to 4.1 * Combining Diacritical Marks Extended (1AB0–1AFF), version 7.0 * Combining Diacritical Marks Supplement (1DC0–1DFF), versions 4.1 to 5.2 * Combining Diacritical Marks for Symbols (20D0–20FF), since version 1.0, with modifications in subsequent versions down to 5.1 * Combining Half Marks (FE20–FE2F), versions 1.0, with modifications in subsequent versions down to 8.0 If it is, we check if the last cell appears to contain a valid symbol, and if so, we attempt to compose (combine) the last cell with the current character, using utf8proc. If the result is a combined character, we replace the content in the previous cell with the new, combined character. Thus, if you select and copy the printed character, you would get e.g. "\u00e1" instead of "a\u0301". This feature can be disabled. By default, it is enabled if the utf8proc library is found, but can be explicitly disabled, or enabled, with 'meson -Dunicode-combining=disabled|enabled'. --- vt.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) (limited to 'vt.c') diff --git a/vt.c b/vt.c index ab36ac6b..c6f86bdb 100644 --- a/vt.c +++ b/vt.c @@ -5,6 +5,10 @@ #include #include +#if FOOT_UNICODE_COMBINING + #include +#endif + #define LOG_MODULE "vt" #define LOG_ENABLE_DBG 0 #include "log.h" @@ -547,8 +551,56 @@ action_utf8_print(struct terminal *term, uint8_t c) /* Reset VT utf8 state */ term->vt.utf8.idx = 0; - int width = wcwidth(wc); - term_print(term, wc, width); +#if FOOT_UNICODE_COMBINING + /* + * Try to combine with the previous character. + * + * We _could_ try regardless of what 'wc' is. However, for + * performance reasons, we only do it when 'wc' is in a known + * 'combining' range. + * + * TODO: + * - doesn't work when base character is multi-column (we'll only + * see an empty "null" character) + */ + + if (((wc >= 0x0300 && wc <= 0x036F) || /* diacritical marks */ + (wc >= 0x1AB0 && wc <= 0x1AFF) || /* diacritical marks, extended */ + (wc >= 0x1DC0 && wc <= 0x1DFF) || /* diacritical marks, supplement */ + (wc >= 0x20D0 && wc <= 0x20FF) || /* diacritical marks, for symbols */ + (wc >= 0xFE20 && wc <= 0xFE2F)) /* half marks */ + && term->grid->cursor.point.col > 0) + { + int base_col = term->grid->cursor.point.col; + if (!term->grid->cursor.lcf) + base_col--; + + assert(base_col >= 0 && base_col < term->cols); + wchar_t base = term->grid->cur_row->cells[base_col].wc; + int base_width = wcwidth(base); + + if (base_width > 0) { + wchar_t composed[] = {base, wc}; + ssize_t composed_length = utf8proc_normalize_utf32( + composed, sizeof(composed) / sizeof(composed[0]), + UTF8PROC_COMPOSE | UTF8PROC_STABLE); + + LOG_DBG("composed = 0x%04x, 0x%04x (length = %zd)", + composed[0], composed[1], composed_length); + + if (composed_length == 1) { + /* Compose succeess - overwrite last cell with + * combined character */ + term->grid->cursor.point.col = base_col; + term->grid->cursor.lcf = false; + term_print(term, composed[0], wcwidth(composed[0])); + return; + } + } + } +#endif /* FOOT_UNICODE_COMBINING */ + + term_print(term, wc, wcwidth(wc)); } static enum state -- cgit v1.3 From e478874dd9cefb08011c46d43deb850e68a4458b Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Mon, 27 Apr 2020 15:06:23 +0200 Subject: term: remove unneeded utf8.left member --- terminal.h | 1 - vt.c | 15 --------------- 2 files changed, 16 deletions(-) (limited to 'vt.c') diff --git a/terminal.h b/terminal.h index 7653daa3..7fcede70 100644 --- a/terminal.h +++ b/terminal.h @@ -135,7 +135,6 @@ struct vt { struct { uint8_t data[4]; uint8_t idx; - uint8_t left; } utf8; struct { uint8_t *data; diff --git a/vt.c b/vt.c index c6f86bdb..0a34a45e 100644 --- a/vt.c +++ b/vt.c @@ -513,27 +513,21 @@ static void action_utf8_2_entry(struct terminal *term, uint8_t c) { term->vt.utf8.idx = 0; - term->vt.utf8.left = 2; term->vt.utf8.data[term->vt.utf8.idx++] = c; - term->vt.utf8.left--; } static void action_utf8_3_entry(struct terminal *term, uint8_t c) { term->vt.utf8.idx = 0; - term->vt.utf8.left = 3; term->vt.utf8.data[term->vt.utf8.idx++] = c; - term->vt.utf8.left--; } static void action_utf8_4_entry(struct terminal *term, uint8_t c) { term->vt.utf8.idx = 0; - term->vt.utf8.left = 4; term->vt.utf8.data[term->vt.utf8.idx++] = c; - term->vt.utf8.left--; } static void @@ -1068,9 +1062,6 @@ static enum state state_utf8_collect_1_switch(struct terminal *term, uint8_t data) { term->vt.utf8.data[term->vt.utf8.idx++] = data; - term->vt.utf8.left--; - - assert(term->vt.utf8.left == 0); action_utf8_print(term, data); return STATE_GROUND; } @@ -1079,9 +1070,6 @@ static enum state state_utf8_collect_2_switch(struct terminal *term, uint8_t data) { term->vt.utf8.data[term->vt.utf8.idx++] = data; - term->vt.utf8.left--; - - assert(term->vt.utf8.left == 1); return STATE_UTF8_COLLECT_1; } @@ -1089,9 +1077,6 @@ static enum state state_utf8_collect_3_switch(struct terminal *term, uint8_t data) { term->vt.utf8.data[term->vt.utf8.idx++] = data; - term->vt.utf8.left--; - - assert(term->vt.utf8.left == 2); return STATE_UTF8_COLLECT_2; } -- cgit v1.3 From 4278af99d2954bd8d87c0db6fa1290663c1d9040 Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Mon, 27 Apr 2020 15:47:44 +0200 Subject: vt: utf8-*-entry: idx is cleared in action_clear() --- vt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'vt.c') diff --git a/vt.c b/vt.c index 0a34a45e..54a9a415 100644 --- a/vt.c +++ b/vt.c @@ -512,21 +512,21 @@ action_put(struct terminal *term, uint8_t c) static void action_utf8_2_entry(struct terminal *term, uint8_t c) { - term->vt.utf8.idx = 0; + assert(term->vt.utf8.idx == 0); term->vt.utf8.data[term->vt.utf8.idx++] = c; } static void action_utf8_3_entry(struct terminal *term, uint8_t c) { - term->vt.utf8.idx = 0; + assert(term->vt.utf8.idx == 0); term->vt.utf8.data[term->vt.utf8.idx++] = c; } static void action_utf8_4_entry(struct terminal *term, uint8_t c) { - term->vt.utf8.idx = 0; + assert(term->vt.utf8.idx == 0); term->vt.utf8.data[term->vt.utf8.idx++] = c; } -- cgit v1.3 From d1fc419e34b125dc99a6f5e5e1818fb1e8e72096 Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Mon, 27 Apr 2020 15:49:07 +0200 Subject: vt: action_utf8_print: idx is cleared in action_clear() --- vt.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'vt.c') diff --git a/vt.c b/vt.c index 54a9a415..3dcb2f57 100644 --- a/vt.c +++ b/vt.c @@ -542,9 +542,6 @@ action_utf8_print(struct terminal *term, uint8_t c) if ((ssize_t)count < 0) wc = 0; - /* Reset VT utf8 state */ - term->vt.utf8.idx = 0; - #if FOOT_UNICODE_COMBINING /* * Try to combine with the previous character. -- cgit v1.3 From 3f3fff768aebe4d1a7822bbe3ecc3e68a3023c8f Mon Sep 17 00:00:00 2001 From: Daniel Eklöf Date: Mon, 27 Apr 2020 15:50:44 +0200 Subject: vt: lazily reset utf8 in action_utf8_*_entry action_clear() is in the super hot code path. Avoid resetting utf8 state there, as utf8 input is relatively uncommon. Instead, reset it when we explicitly enter any of the utf8 collecting states, as this is exactly the point where we need it. --- vt.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'vt.c') diff --git a/vt.c b/vt.c index 3dcb2f57..743580a1 100644 --- a/vt.c +++ b/vt.c @@ -109,7 +109,6 @@ action_clear(struct terminal *term) term->vt.params.idx = 0; term->vt.private[0] = 0; term->vt.private[1] = 0; - term->vt.utf8.idx = 0; } static void @@ -512,21 +511,21 @@ action_put(struct terminal *term, uint8_t c) static void action_utf8_2_entry(struct terminal *term, uint8_t c) { - assert(term->vt.utf8.idx == 0); + term->vt.utf8.idx = 0; term->vt.utf8.data[term->vt.utf8.idx++] = c; } static void action_utf8_3_entry(struct terminal *term, uint8_t c) { - assert(term->vt.utf8.idx == 0); + term->vt.utf8.idx = 0; term->vt.utf8.data[term->vt.utf8.idx++] = c; } static void action_utf8_4_entry(struct terminal *term, uint8_t c) { - assert(term->vt.utf8.idx == 0); + term->vt.utf8.idx = 0; term->vt.utf8.data[term->vt.utf8.idx++] = c; } -- cgit v1.3