Add UTF8/UTF16 codepoint count via new simdutf submodule

Doy-lee · Doy-lee · commit f83433723817 · 2025-08-29T15:25:57.000+10:00
diff --git a/.gitmodules b/.gitmodules
@@ -22,3 +22,6 @@
 [submodule "external/oxen-logging"]
 	path = external/oxen-logging
 	url = https://github.com/oxen-io/oxen-logging.git
+[submodule "external/simdutf"]
+	path = external/simdutf
+	url = git@github.com:simdutf/simdutf.git
diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt
@@ -193,3 +193,14 @@ libsession_static_bundle(libzstd_static)
 set(JSON_BuildTests OFF CACHE INTERNAL "")
 set(JSON_Install ON CACHE INTERNAL "") # Required to export targets that we use
 libsession_system_or_submodule(NLOHMANN nlohmann_json nlohmann_json>=3.7.0 nlohmann-json)
+
+set(JSON_BuildTests OFF CACHE INTERNAL "")
+set(JSON_Install ON CACHE INTERNAL "") # Required to export targets that we use
+
+function(simdutf_subdir)
+    set(SIMDUTF_TESTS OFF CACHE BOOL "")
+    set(SIMDUTF_TOOLS OFF CACHE BOOL "")
+    set(BUILD_SHARED_LIBS OFF)
+    add_subdirectory(simdutf)
+endfunction()
+simdutf_subdir()
diff --git a/external/simdutf b/external/simdutf
@@ -0,0 +1 @@
+Subproject commit 7b3f5afcae322391a03736809ef6eea0c2934388
diff --git a/include/session/session_protocol.h b/include/session/session_protocol.h
@@ -14,17 +14,12 @@ extern "C" {
 #endif
 
 enum {
-    /// TODO: This comment needs to be updated to be _codepoints_ once libsession implements the
-    /// character count for the platforms. Currently they use code units but it should be
-    /// codepoints. This allows the platforms to use their native text representation up until the
-    /// API boundary where they will convert to UTF8 to have it managed by libsession.
-
-    /// Maximum number of UTF16 code units that a standard message can use. If the message exceeds
+    /// Maximum number of UTF16 code points that a standard message can use. If the message exceeds
     /// this then the message must activate the higher character limit feature provided by Session
     /// Pro which allows messages up to 10k characters.
     PRO_STANDARD_CHARACTER_LIMIT = 2'000,
 
-    /// Maximum number of UTF16 code units that a Session Pro entitled user can send in a message.
+    /// Maximum number of UTF16 code points that a Session Pro entitled user can send in a message.
     /// This is not used in the codebase, but is provided for convenience to centralise protocol
     /// definitions for users of the library to consume.
     PRO_HIGHER_CHARACTER_LIMIT = 10'000,
@@ -118,21 +113,58 @@ typedef struct session_protocol_encrypted_for_destination {
     size_t error_len_incl_null_terminator;
 } session_protocol_encrypted_for_destination;
 
-/// API: session_protocol/session_protocol_get_pro_features_for_msg
+typedef struct session_protocol_pro_features_for_msg {
+    bool success;
+    string8 error;
+    PRO_FEATURES features;
+    size_t codepoint_count;
+} session_protocol_pro_features_for_msg;
+
+/// API: session_protocol/session_protocol_get_pro_features_for_utf8
+///
+/// Determine the Pro features that are used in a given UTF8 message.
+///
+/// Inputs:
+/// - `utf8` -- the utf8 string to count the number of codepoints in to determine if it needs the
+///   higher character limit available in Session Pro
+/// - `utf8_size` -- the number of code units (aka. bytes) the string has
+/// - `flags` -- extra pro features that are known by clients that they wish to be activated on
+///   this message
+///
+/// Outputs:
+/// - `success` -- True if the message was evaluated successfully for PRO features false otherwise.
+///   When false, all fields except for `error` should be ignored from the result object.
+/// - `error` -- If `success` is false, this is populated with an error code describing the error,
+//    otherwise it's empty.
+/// - `features` -- Session Pro feature flags suitable for writing directly into the protobuf
+///   `ProMessage` in `Content`
+/// - `codepoint_count` -- Counts the number of unicode codepoints that were in the message.
+LIBSESSION_EXPORT
+session_protocol_pro_features_for_msg session_protocol_pro_features_for_utf8(
+        char const* utf8, size_t utf8_size, PRO_EXTRA_FEATURES extra);
+
+/// API: session_protocol/session_protocol_get_pro_features_for_utf16
 ///
-/// Determine the Pro features that are used in a given conversation message.
+/// Determine the Pro features that are used in a given UTF16 message.
 ///
 /// Inputs:
-/// - `msg_size` -- the size of the message in UTF16 code units to determine if the message requires
-///   access to the higher character limit available in Session Pro
+/// - `utf8` -- the utf16 string to count the number of codepoints in to determine if it needs the
+///   higher character limit available in Session Pro
+/// - `utf16_size` -- the number of code units (aka. bytes) the string has
 /// - `flags` -- extra pro features that are known by clients that they wish to be activated on
 ///   this message
 ///
 /// Outputs:
-/// - Session Pro feature flags suitable for writing directly into the protobuf `ProMessage` in
-///   `Content`
+/// - `success` -- True if the message was evaluated successfully for PRO features false otherwise.
+///   When false, all fields except for `error` should be ignored from the result object.
+/// - `error` -- If `success` is false, this is populated with an error code describing the error,
+//    otherwise it's empty.
+/// - `features` -- Session Pro feature flags suitable for writing directly into the protobuf
+///   `ProMessage` in `Content`
+/// - `codepoint_count` -- Counts the number of unicode codepoints that were in the message.
 LIBSESSION_EXPORT
-PRO_FEATURES session_protocol_get_pro_features_for_msg(size_t msg_size, PRO_EXTRA_FEATURES flags);
+session_protocol_pro_features_for_msg session_protocol_pro_features_for_utf16(
+        uint16_t const* utf16, size_t utf16_size, PRO_EXTRA_FEATURES extra);
 
 /// API: session_protocol_encrypt_for_1o1
 ///
diff --git a/include/session/session_protocol.hpp b/include/session/session_protocol.hpp
@@ -163,7 +163,35 @@ struct DecryptEnvelopeKey {
     std::span<std::span<const uint8_t>> ed25519_privkeys;
 };
 
-/// API: session_protocol/get_pro_features_for_msg
+struct ProFeaturesForMsg
+{
+    bool success;
+    std::string_view error;
+    PRO_FEATURES features;
+    size_t codepoint_count;
+};
+
+/// API: session_protocol/get_pro_features_for_utf8
+///
+/// Determine the Pro features that are used in a given conversation message.
+///
+/// Inputs:
+/// - `msg_size` -- the size of the message in UTF8 code units to determine if the message requires
+///   access to the higher character limit available in Session Pro
+/// - `flags` -- extra pro features that are known by clients that they wish to be activated on
+///   this message
+///
+/// Outputs:
+/// - `success` -- True if the message was evaluated successfully for PRO features false otherwise.
+///   When false, all fields except for `error` should be ignored from the result object.
+/// - `error` -- If `success` is false, this is populated with an error code describing the error,
+//    otherwise it's empty.
+/// - `features` -- Session Pro feature flags suitable for writing directly into the protobuf
+///   `ProMessage` in `Content`
+/// - `codepoint_count` -- Counts the number of unicode codepoints that were in the message.
+ProFeaturesForMsg get_pro_features_for_utf8(char const *utf8, size_t utf8_size, PRO_EXTRA_FEATURES flags);
+
+/// API: session_protocol/get_pro_features_for_utf16
 ///
 /// Determine the Pro features that are used in a given conversation message.
 ///
@@ -174,9 +202,15 @@ struct DecryptEnvelopeKey {
 ///   this message
 ///
 /// Outputs:
-/// - Session Pro feature flags suitable for writing directly into the protobuf `ProMessage` in
-///   `Content`
-PRO_FEATURES get_pro_features_for_msg(size_t msg_size, PRO_EXTRA_FEATURES flags);
+/// - `success` -- True if the message was evaluated successfully for PRO features false otherwise.
+///   When false, all fields except for `error` should be ignored from the result object.
+/// - `error` -- If `success` is false, this is populated with an error code describing the error,
+//    otherwise it's empty.
+/// - `features` -- Session Pro feature flags suitable for writing directly into the protobuf
+///   `ProMessage` in `Content`
+/// - `codepoint_count` -- Counts the number of unicode codepoints that were in the message.
+ProFeaturesForMsg get_pro_features_for_utf16(
+        char16_t const* utf16, size_t utf8_size, PRO_EXTRA_FEATURES flags);
 
 /// API: session_protocol/encrypt_for_1o1
 ///
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -94,6 +94,7 @@ target_link_libraries(crypto
     libsodium::sodium-internal
     nlohmann_json::nlohmann_json
     libsession::protos
+    simdutf
 )
 
 target_link_libraries(config
diff --git a/src/session_protocol.cpp b/src/session_protocol.cpp
@@ -1,6 +1,7 @@
 #include <fmt/core.h>
 #include <oxenc/hex.h>
 #include <session/config/groups/keys.h>
+#include <simdutf.h>
 #include <sodium/crypto_sign_ed25519.h>
 #include <sodium/randombytes.h>
 
@@ -16,21 +17,47 @@
 #include "WebSocketResources.pb.h"
 #include "session/export.h"
 
-namespace session {
-
-PRO_FEATURES get_pro_features_for_msg(size_t msg_size, PRO_EXTRA_FEATURES extra) {
-    PRO_FEATURES result = PRO_FEATURES_NIL;
-
-    if (msg_size > PRO_STANDARD_CHARACTER_LIMIT)
-        result |= PRO_FEATURES_10K_CHARACTER_LIMIT;
+namespace {
+session::ProFeaturesForMsg pro_features_for_utf8_or_16(
+        const void* utf, size_t utf_size, PRO_EXTRA_FEATURES extra, bool is_utf8) {
+    session::ProFeaturesForMsg result = {};
+    simdutf::result validate = is_utf8 ? simdutf::validate_utf8_with_errors(
+                                                 reinterpret_cast<const char*>(utf), utf_size)
+                                       : simdutf::validate_utf16_with_errors(
+                                                 reinterpret_cast<const char16_t*>(utf), utf_size);
+    if (validate.is_ok()) {
+        result.success = true;
+        result.codepoint_count =
+                is_utf8 ? simdutf::count_utf8(reinterpret_cast<const char*>(utf), utf_size)
+                        : simdutf::count_utf16(reinterpret_cast<const char16_t*>(utf), utf_size);
+        if (result.codepoint_count > PRO_STANDARD_CHARACTER_LIMIT)
+            result.features |= PRO_FEATURES_10K_CHARACTER_LIMIT;
+
+        if (extra & PRO_EXTRA_FEATURES_ANIMATED_AVATAR)
+            result.features |= PRO_FEATURES_ANIMATED_AVATAR;
+
+        if (extra & PRO_EXTRA_FEATURES_PRO_BADGE)
+            result.features |= PRO_FEATURES_PRO_BADGE;
+
+        assert((result.features & ~PRO_FEATURES_ALL) == 0);
+    } else {
+        result.error = simdutf::error_to_string(validate.error);
+    }
+    return result;
+}
+};  // namespace
 
-    if (extra & PRO_EXTRA_FEATURES_ANIMATED_AVATAR)
-        result |= PRO_FEATURES_ANIMATED_AVATAR;
+namespace session {
 
-    if (extra & PRO_EXTRA_FEATURES_PRO_BADGE)
-        result |= PRO_FEATURES_PRO_BADGE;
+ProFeaturesForMsg pro_features_for_utf8(
+        const char* utf, size_t utf_size, PRO_EXTRA_FEATURES extra) {
+    ProFeaturesForMsg result = pro_features_for_utf8_or_16(utf, utf_size, extra, /*is_utf8*/ true);
+    return result;
+}
 
-    assert((result & ~PRO_FEATURES_ALL) == 0);
+ProFeaturesForMsg pro_features_for_utf16(
+        const uint16_t* utf, size_t utf_size, PRO_EXTRA_FEATURES extra) {
+    ProFeaturesForMsg result = pro_features_for_utf8_or_16(utf, utf_size, extra, /*is_utf8*/ false);
     return result;
 }
 
@@ -390,8 +417,10 @@ DecryptedEnvelope decrypt_envelope(
         // the source is a Session public key (see: encrypt_for_destination)
         const std::string& source = envelope.source();
         if (source.size() != result.envelope.source.max_size())
-            throw std::runtime_error(fmt::format(
-                    "Parse envelope failed, source had unexpected size ({} bytes)", source.size()));
+            throw std::runtime_error(
+                    fmt::format(
+                            "Parse envelope failed, source had unexpected size ({} bytes)",
+                            source.size()));
         std::memcpy(result.envelope.source.data(), source.data(), source.size());
         result.envelope.flags |= ENVELOPE_FLAGS_SOURCE;
     }
@@ -546,8 +575,28 @@ DecryptedEnvelope decrypt_envelope(
 using namespace session;
 
 LIBSESSION_C_API
-PRO_FEATURES session_protocol_get_pro_features_for_msg(size_t msg_size, PRO_EXTRA_FEATURES flags) {
-    PRO_FEATURES result = get_pro_features_for_msg(msg_size, flags);
+session_protocol_pro_features_for_msg session_protocol_pro_features_for_utf8(
+        const char* utf, size_t utf_size, PRO_EXTRA_FEATURES extra) {
+    ProFeaturesForMsg result_cpp = pro_features_for_utf8_or_16(utf, utf_size, extra, /*is_utf8*/ true);
+    session_protocol_pro_features_for_msg result = {
+            .success = result_cpp.success,
+            .error = {const_cast<char*>(result_cpp.error.data()), result_cpp.error.size()},
+            .features = result_cpp.features,
+            .codepoint_count = result_cpp.codepoint_count,
+    };
+    return result;
+}
+
+LIBSESSION_C_API
+session_protocol_pro_features_for_msg session_protocol_pro_features_for_utf16(
+        const uint16_t* utf, size_t utf_size, PRO_EXTRA_FEATURES extra) {
+    ProFeaturesForMsg result_cpp = pro_features_for_utf8_or_16(utf, utf_size, extra, /*is_utf8*/ false);
+    session_protocol_pro_features_for_msg result = {
+            .success = result_cpp.success,
+            .error = {const_cast<char*>(result_cpp.error.data()), result_cpp.error.size()},
+            .features = result_cpp.features,
+            .codepoint_count = result_cpp.codepoint_count,
+    };
     return result;
 }
 
diff --git a/tests/test_session_protocol.cpp b/tests/test_session_protocol.cpp
@@ -95,21 +95,52 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") {
     // Do tests that require no setup
     SECTION("Ensure get pro fetaures detects large message") {
         // Try a message below the size threshold
-        PRO_FEATURES features = session_protocol_get_pro_features_for_msg(
-                PRO_STANDARD_CHARACTER_LIMIT,
-                PRO_EXTRA_FEATURES_PRO_BADGE | PRO_EXTRA_FEATURES_ANIMATED_AVATAR);
-        REQUIRE(features == (PRO_FEATURES_PRO_BADGE | PRO_FEATURES_ANIMATED_AVATAR));
+        {
+            auto msg = std::string(PRO_STANDARD_CHARACTER_LIMIT, 'a');
+            session_protocol_pro_features_for_msg pro_msg =
+                    session_protocol_pro_features_for_utf8(
+                            msg.data(),
+                            msg.size(),
+                            PRO_EXTRA_FEATURES_PRO_BADGE | PRO_EXTRA_FEATURES_ANIMATED_AVATAR);
+            REQUIRE(pro_msg.success);
+            REQUIRE(pro_msg.features == (PRO_FEATURES_PRO_BADGE | PRO_FEATURES_ANIMATED_AVATAR));
+            REQUIRE(pro_msg.codepoint_count == msg.size());
+        }
+
+        // Try an invalid message
+        {
+            std::string_view msg = "\xFF";
+            session_protocol_pro_features_for_msg pro_msg = session_protocol_pro_features_for_utf8(
+                    msg.data(), msg.size(), PRO_FEATURES_NIL);
+            REQUIRE(!pro_msg.success);
+            REQUIRE(pro_msg.error.size);
+        }
 
         // Try a message exceeding the size threshold
-        features = session_protocol_get_pro_features_for_msg(
-                PRO_STANDARD_CHARACTER_LIMIT + 1,
-                PRO_EXTRA_FEATURES_PRO_BADGE | PRO_EXTRA_FEATURES_ANIMATED_AVATAR);
-        REQUIRE(features == (PRO_FEATURES_10K_CHARACTER_LIMIT | PRO_FEATURES_PRO_BADGE |
-                             PRO_FEATURES_ANIMATED_AVATAR));
+        {
+            auto msg = std::string(PRO_STANDARD_CHARACTER_LIMIT + 1, 'a');
+            session_protocol_pro_features_for_msg pro_msg = session_protocol_pro_features_for_utf8(
+                    msg.data(),
+                    msg.size(),
+                    PRO_EXTRA_FEATURES_PRO_BADGE | PRO_EXTRA_FEATURES_ANIMATED_AVATAR);
+            REQUIRE(pro_msg.success);
+            REQUIRE(pro_msg.features == (PRO_FEATURES_10K_CHARACTER_LIMIT | PRO_FEATURES_PRO_BADGE |
+                                         PRO_FEATURES_ANIMATED_AVATAR));
+            REQUIRE(pro_msg.codepoint_count == msg.size());
+        }
 
         // Try asking for just one extra feature
-        features = session_protocol_get_pro_features_for_msg(100, PRO_EXTRA_FEATURES_PRO_BADGE);
-        REQUIRE(features == PRO_FEATURES_PRO_BADGE);
+        {
+            auto msg = std::string(PRO_STANDARD_CHARACTER_LIMIT, 'a');
+            session_protocol_pro_features_for_msg pro_msg =
+                    session_protocol_pro_features_for_utf8(
+                            msg.data(),
+                            msg.size(),
+                            PRO_EXTRA_FEATURES_PRO_BADGE);
+            REQUIRE(pro_msg.success);
+            REQUIRE(pro_msg.features == PRO_FEATURES_PRO_BADGE);
+            REQUIRE(pro_msg.codepoint_count == msg.size());
+        }
     }
 
     // Tests that require some setup code
@@ -367,17 +398,17 @@ TEST_CASE("Session protocol helpers C API", "[session-protocol][helpers]") {
         std::string large_message;
         large_message.resize(PRO_STANDARD_CHARACTER_LIMIT + 1);
 
-        PRO_FEATURES features =
-                get_pro_features_for_msg(large_message.size(), PRO_EXTRA_FEATURES_PRO_BADGE);
-        REQUIRE(features == (PRO_FEATURES_10K_CHARACTER_LIMIT | PRO_FEATURES_PRO_BADGE));
+        session_protocol_pro_features_for_msg pro_msg = session_protocol_pro_features_for_utf8(
+                large_message.data(), large_message.size(), PRO_EXTRA_FEATURES_PRO_BADGE);
+        REQUIRE(pro_msg.features == (PRO_FEATURES_10K_CHARACTER_LIMIT | PRO_FEATURES_PRO_BADGE));
 
         SerialisedProtobufContentWithProForTesting protobuf_content_with_pro_and_features =
                 build_protobuf_content_with_session_pro(
                         /*data_body*/ large_message,
                         /*user_rotating_privkey*/ user_pro_ed_sk,
                         /*pro_backend_privkey*/ pro_backend_ed_sk,
                         /*pro_expiry_unix_ts*/ timestamp_s,
-                        features);
+                        pro_msg.features);
 
         // Encrypt content
         session_protocol_encrypted_for_destination encrypt_result =

Original file line number	Diff line number	Diff line change
`@@ -94,6 +94,7 @@ target_link_libraries(crypto`
`94`	`94`	`libsodium::sodium-internal`
`95`	`95`	`nlohmann_json::nlohmann_json`
`96`	`96`	`libsession::protos`
	`97`	`+ simdutf`
`97`	`98`	`)`
`98`	`99`
`99`	`100`	`target_link_libraries(config`