From ba1810606fa5d82dcf9daece818b43098b9ebd1e Mon Sep 17 00:00:00 2001 From: Julius Pfrommer Date: Wed, 18 Dec 2024 11:10:39 +0100 Subject: [PATCH] feat(deps): Add lightweight utf8 de/encoding as a standalone implementation --- deps/README.md | 1 + deps/utf8.c | 67 ++++++++++++++++++++++++++++++++++++++++ deps/utf8.h | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 deps/utf8.c create mode 100644 deps/utf8.h diff --git a/deps/README.md b/deps/README.md index e47742bb7..95232b8f6 100644 --- a/deps/README.md +++ b/deps/README.md @@ -21,3 +21,4 @@ The following third party libraries may be included -- depending on the activate | mqtt-c | MIT | a portable MQTT client in C | | dtoa | BSL (Boost) | Printing of float numbers | | mp_printf | MIT | Our version of github:mpaland/printf | +| utf8 | MPL 2.0 | Lightweight utf8 de/encoding | diff --git a/deps/utf8.c b/deps/utf8.c new file mode 100644 index 000000000..ba6e0e582 --- /dev/null +++ b/deps/utf8.c @@ -0,0 +1,67 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 2024 (c) Fraunhofer IOSB (Author: Julius Pfrommer) + */ + +#include "utf8.h" + +#define UTF_PARSE_BYTE(pos) do { \ + byte = str[pos]; \ + if(UTF_UNLIKELY(byte < 0x80 || byte > 0xBF)) \ + return 0; /* Not a continuation byte */ \ + *codepoint = (*codepoint << 6) + (byte & 0x3F); \ + } while(0) + +unsigned +utf8_to_codepoint(const unsigned char *str, size_t len, unsigned *codepoint) { + /* Ensure there is a character to read */ + if(UTF_UNLIKELY(len == 0)) + return 0; + + *codepoint = str[0]; + if(UTF_LIKELY(*codepoint < 0x80)) + return 1; /* Normal ASCII */ + + if(UTF_UNLIKELY(*codepoint <= 0xC1)) + return 0; /* Continuation byte not allowed here */ + + unsigned count; + unsigned char byte; + if(*codepoint <= 0xDF) { /* 2-byte sequence */ + if(len < 2) + return 0; + count = 2; + *codepoint &= 0x1F; + UTF_PARSE_BYTE(1); + if(UTF_UNLIKELY(*codepoint < 0x80)) + return 0; /* Too small for the encoding length */ + } else if(*codepoint <= 0xEF) { /* 3-byte sequence */ + if(len < 3) + return 0; + count = 3; + *codepoint &= 0xF; + UTF_PARSE_BYTE(1); + UTF_PARSE_BYTE(2); + if(UTF_UNLIKELY(*codepoint < 0x800)) + return 0; /* Too small for the encoding length */ + } else if(*codepoint <= 0xF4) { /* 4-byte sequence */ + if(len < 4) + return 0; + count = 4; + *codepoint &= 0x7; + UTF_PARSE_BYTE(1); + UTF_PARSE_BYTE(2); + UTF_PARSE_BYTE(3); + if(UTF_UNLIKELY(*codepoint < 0x10000)) + return 0; /* Too small for the encoding length */ + } else { + return 0; /* Invalid utf8 encoding */ + } + + if(UTF_UNLIKELY(*codepoint > 0x10FFFF)) + return 0; /* Not in the Unicode range */ + + return count; +} diff --git a/deps/utf8.h b/deps/utf8.h new file mode 100644 index 000000000..9ba0b539d --- /dev/null +++ b/deps/utf8.h @@ -0,0 +1,83 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 2024 (c) Fraunhofer IOSB (Author: Julius Pfrommer) + */ + +#ifndef UTF8_H_ +#define UTF8_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _MSC_VER +# define UTF_INLINE __inline +#else +# define UTF_INLINE inline +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define UTF_LIKELY(x) __builtin_expect((x), 1) +# define UTF_UNLIKELY(x) __builtin_expect((x), 0) +#else +# define UTF_LIKELY(x) (x) +# define UTF_UNLIKELY(x) (x) +#endif + +/* Extract the next utf8 codepoint from the buffer. Returns the length (1-4) of + * the codepoint encoding or 0 upon an error. */ +unsigned +utf8_to_codepoint(const unsigned char *str, size_t len, unsigned *codepoint); + +/* Encodes the codepoint in utf8. The string needs to have enough space (at most + * four byte) available. Returns the encoding length. */ +static UTF_INLINE unsigned +utf8_from_codepoint(unsigned char *str, unsigned codepoint) { + if(UTF_LIKELY(codepoint <= 0x7F)) { /* Plain ASCII */ + str[0] = (unsigned char)codepoint; + return 1; + } + if(UTF_LIKELY(codepoint <= 0x07FF)) { /* 2-byte unicode */ + str[0] = (unsigned char)(((codepoint >> 6) & 0x1F) | 0xC0); + str[1] = (unsigned char)(((codepoint >> 0) & 0x3F) | 0x80); + return 2; + } + if(UTF_LIKELY(codepoint <= 0xFFFF)) { /* 3-byte unicode */ + str[0] = (unsigned char)(((codepoint >> 12) & 0x0F) | 0xE0); + str[1] = (unsigned char)(((codepoint >> 6) & 0x3F) | 0x80); + str[2] = (unsigned char)(((codepoint >> 0) & 0x3F) | 0x80); + return 3; + } + if(UTF_LIKELY(codepoint <= 0x10FFFF)) { /* 4-byte unicode */ + str[0] = (unsigned char)(((codepoint >> 18) & 0x07) | 0xF0); + str[1] = (unsigned char)(((codepoint >> 12) & 0x3F) | 0x80); + str[2] = (unsigned char)(((codepoint >> 6) & 0x3F) | 0x80); + str[3] = (unsigned char)(((codepoint >> 0) & 0x3F) | 0x80); + return 4; + } + return 0; /* Not a unicode codepoint */ +} + +/* Returns the encoding length of the codepoint */ +static UTF_INLINE unsigned +utf8_length(unsigned codepoint) { + if(UTF_LIKELY(codepoint <= 0x7F)) + return 1; /* Plain ASCII */ + if(UTF_LIKELY(codepoint <= 0x07FF)) + return 2; /* 2-byte unicode */ + if(UTF_LIKELY(codepoint <= 0xFFFF)) + return 3; /* 3-byte unicode */ + if(UTF_LIKELY(codepoint <= 0x10FFFF)) + return 4; /* 4-byte unicode */ + return 0; /* Not a unicode codepoint */ +} + +#ifdef __cplusplus +} +#endif + +#endif /* UTF8_H_ */