From adf710ad2348487db66c2f8b3998c65bc7b1e50f Mon Sep 17 00:00:00 2001
From: JB1 <54802325+jbender0@users.noreply.github.com>
Date: Mon, 20 Jul 2026 18:03:06 +0700
Subject: [PATCH] Replace mb_str_split with str_split for key processing

DoubleArray::commonPrefixSearch() in Precompiled.php:147-170 is a byte-level double-array trie (ported from sentencepiece's C++ normalizer), but it iterates with mb_str_split() (character-wise) instead of byte-wise, then calls ord() on multi-byte characters - triggering a deprecation warning and, more importantly, silently truncating multi-byte UTF-8 input to its first byte, which breaks the trie lookup for any non-ASCII text (e.g. accented characters, non-Latin scripts) going through this normalizer.
---
 src/Normalizers/Precompiled.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Normalizers/Precompiled.php b/src/Normalizers/Precompiled.php
index 72b9a90..54ec6b6 100644
--- a/src/Normalizers/Precompiled.php
+++ b/src/Normalizers/Precompiled.php
@@ -152,13 +152,13 @@ public function commonPrefixSearch($key): array
         $unit = $this->array[$node_pos];
         $node_pos ^= $this->offset($unit);
 
-        foreach (mb_str_split($key) as $c) {
+        foreach (str_split($key) as $c) {
             if (ord($c) === 0) {
                 break;
             }
             $node_pos ^= ord($c);
             $unit = $this->array[$node_pos];
-            if ($this->label($unit) !== mb_ord($c)) {
+            if ($this->label($unit) !== ord($c)) {
                 return $results;
             }
             $node_pos ^= $this->offset($unit);