From 2f4cb88d5c60f725323739300bb49dfa8923e7d5 Mon Sep 17 00:00:00 2001 From: Albert Cervin Date: Wed, 2 Nov 2022 22:20:04 +0100 Subject: =?UTF-8?q?=F0=9F=8E=89=20And=20so=20it=20begins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utf8.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/utf8.c (limited to 'src/utf8.c') diff --git a/src/utf8.c b/src/utf8.c new file mode 100644 index 0000000..3afef40 --- /dev/null +++ b/src/utf8.c @@ -0,0 +1,42 @@ +#include "utf8.h" + +#include + +bool utf8_byte_is_unicode_start(uint8_t byte) { return (byte & 0xc0) == 0xc0; } +bool utf8_byte_is_unicode_continuation(uint8_t byte) { + return utf8_byte_is_unicode(byte) && !utf8_byte_is_unicode_start(byte); +} +bool utf8_byte_is_unicode(uint8_t byte) { return (byte & 0x80) != 0x0; } +bool utf8_byte_is_ascii(uint8_t byte) { return !utf8_byte_is_unicode(byte); } + +// TODO: grapheme clusters, this returns the number of unicode code points +uint32_t utf8_nchars(uint8_t *bytes, uint32_t nbytes) { + uint32_t nchars = 0; + for (uint32_t bi = 0; bi < nbytes; ++bi) { + if (utf8_byte_is_ascii(bytes[bi]) || utf8_byte_is_unicode_start(bytes[bi])) + ++nchars; + } + return nchars; +} + +// TODO: grapheme clusters, this uses the number of unicode code points +uint32_t utf8_nbytes(uint8_t *bytes, uint32_t nchars) { + uint32_t bi = 0; + uint32_t chars = 0; + while (chars < nchars) { + uint8_t byte = bytes[bi]; + if (utf8_byte_is_unicode_start(byte)) { + ++chars; + + // length of char is the number of leading ones + // flip it and count number of leading zeros + uint8_t invb = ~byte; + bi += __builtin_clz((uint32_t)invb) - 24; + } else { + ++chars; + ++bi; + } + } + + return bi; +} -- cgit v1.2.3