From b631d7e3d0084aa6f51d04794bc0bd196ed3fc38 Mon Sep 17 00:00:00 2001 From: uo278036 Date: Mon, 16 Nov 2026 10:51:56 +0100 Subject: [PATCH 1/3] fix(memory): keep combining marks in words in InMemoryMemoryService search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The in-memory service extracted words with \w+, which does not match combining marks, so Thai and Devanagari words were split at their vowel and tone marks. The pieces were then matched as substrings, and a search for ข้าว also returned unrelated Thai text. Tokenize the same way SqliteMemoryService does: combining marks are word characters, and a mark with no letter before it is dropped. Fixes #8168 --- .../adk/memory/in_memory_memory_service.py | 42 +++++++++++++++---- .../memory/test_in_memory_memory_service.py | 9 ++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/google/adk/memory/in_memory_memory_service.py b/src/google/adk/memory/in_memory_memory_service.py index 11c974165bd..f25ec904e67 100644 --- a/src/google/adk/memory/in_memory_memory_service.py +++ b/src/google/adk/memory/in_memory_memory_service.py @@ -16,7 +16,6 @@ from collections.abc import Mapping from collections.abc import Sequence import itertools -import re import threading from typing import TYPE_CHECKING import unicodedata @@ -46,10 +45,35 @@ def _is_latin(c: str) -> bool: return unicodedata.name(c, '').startswith('LATIN') +def _is_mark(c: str) -> bool: + return unicodedata.category(c).startswith('M') + + +# Combining marks count as word characters: \w alone would split a word at +# them, such as the Thai vowel and tone marks in ข้าว. This matches the +# tokenizer of SqliteMemoryService. +def _is_word_char(c: str) -> bool: + return c.isalnum() or c == '_' or _is_mark(c) + + +def _find_words(text: str) -> list[str]: + """Returns the words of an NFC-normalized text, marks included.""" + words = [] + for is_word, group in itertools.groupby(text, _is_word_char): + if not is_word: + continue + # Marks with no letter before them are dropped: the emoji variation + # selector U+FE0F is a mark, and as a word it would match any emoji. + word = ''.join(itertools.dropwhile(_is_mark, group)) + if word: + words.append(word) + return words + + def _extract_words_lower(text: str) -> set[str]: """Extracts Unicode-aware tokens from a string in lowercase.""" text = unicodedata.normalize('NFC', text) - return set(word.lower() for word in re.findall(r'\w+', text)) + return set(word.lower() for word in _find_words(text)) def _extract_searchable_words(text: str) -> set[str]: @@ -57,7 +81,7 @@ def _extract_searchable_words(text: str) -> set[str]: The tokens of _extract_words_lower, plus, for a token that mixes Latin and non-Latin scripts, each of its single-script runs. Japanese and Chinese are - written without spaces, so 私はPythonを使う is a single \w+ token and a query + written without spaces, so 私はPythonを使う is a single token and a query for Python matches nothing. Splitting on the boundary between Latin and non-Latin scripts makes the embedded Latin word a token of its own, while still keeping a partial word such as thon from matching. Queries and events @@ -68,15 +92,17 @@ def _extract_searchable_words(text: str) -> set[str]: text: The text of a query or an event. """ words: set[str] = set() - for word in re.findall(r'\w+', unicodedata.normalize('NFC', text)): + for word in _find_words(unicodedata.normalize('NFC', text)): words.add(word.lower()) if not word.isascii(): # Split before lowercasing: "İ".lower() is "i" plus U+0307, which is # not Latin, so splitting afterwards yields a stray "i" word. - words.update( - ''.join(group).lower() - for _, group in itertools.groupby(word, _is_latin) - ) + for _, group in itertools.groupby(word, _is_latin): + # A combining mark on a Latin letter is not Latin itself, so it can + # form a run of its own; like a leading mark, it is not a word. + run = ''.join(itertools.dropwhile(_is_mark, group)) + if run: + words.add(run.lower()) return words diff --git a/tests/unittests/memory/test_in_memory_memory_service.py b/tests/unittests/memory/test_in_memory_memory_service.py index 645385429b9..9780abd002e 100644 --- a/tests/unittests/memory/test_in_memory_memory_service.py +++ b/tests/unittests/memory/test_in_memory_memory_service.py @@ -435,6 +435,15 @@ async def test_search_memory_does_not_collide_on_slash_in_identifiers(): ('私と犬', 'Pythonと', 0), # "İ".lower() is "i" plus U+0307: no stray "i" word to match on ('I like cats.', 'İstanbul旅行', 0), + # Combining marks are part of a word: Thai vowel and tone marks, + # Devanagari vowel signs + ('ข้าวผัดอร่อยมาก', 'ข้าว', 1), + ('ขอบคุณครับ', 'ข้าว', 0), + ('मुझे हिंदी पसंद है', 'हिंदी', 1), + ('मैं हैदराबाद में रहता हूँ', 'हिंदी', 0), + # A mark with no letter before it, such as the emoji variation + # selector U+FE0F, is not a word + ('I love cats ❤️', 'Dogs ❤️', 0), # Latin partial-word must NOT match (regression guard) ('I like to code in Python.', 'thon', 0), ('私はPythonでADKを使っています', 'thon', 0), From 05987f94629a37d385ee1de51ad8a6665d61bd72 Mon Sep 17 00:00:00 2001 From: uo278036 Date: Mon, 16 Nov 2026 10:51:56 +0100 Subject: [PATCH 2/3] test(memory): cover Bengali, Tamil, Latin+Thai and keycap emoji in search Adds cases to test_search_memory_non_latin for Bengali and Tamil vowel signs and virama, a Latin word inside Thai text, and the keycap emoji 1 + U+FE0F + U+20E3. The Bengali, Tamil and Latin+Thai negative cases fail without the combining-mark fix. --- .../memory/test_in_memory_memory_service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unittests/memory/test_in_memory_memory_service.py b/tests/unittests/memory/test_in_memory_memory_service.py index 9780abd002e..5b390a94f1d 100644 --- a/tests/unittests/memory/test_in_memory_memory_service.py +++ b/tests/unittests/memory/test_in_memory_memory_service.py @@ -444,6 +444,18 @@ async def test_search_memory_does_not_collide_on_slash_in_identifiers(): # A mark with no letter before it, such as the emoji variation # selector U+FE0F, is not a word ('I love cats ❤️', 'Dogs ❤️', 0), + # Bengali and Tamil vowel signs and virama + ('আমি ভাত খাই', 'ভাত', 1), + ('ভালো আছি', 'ভাত', 0), + ('தமிழ் எனக்கு பிடிக்கும்', 'தமிழ்', 1), + ('நான் சாதம் சாப்பிடுகிறேன்', 'தமிழ்', 0), + # Latin word inside Thai text + ('ฉันใช้Pythonเขียนโปรแกรม', 'Python', 1), + ('ฉันใช้Pythonเขียนโปรแกรม', 'เขียน', 1), + ('ฉันใช้Pythonเขียนโปรแกรม', 'ข้าว', 0), + # Keycap emoji: 1, U+FE0F and the enclosing keycap mark U+20E3 + ('Room 1️⃣ is free', '1️⃣', 1), + ('Room 2 is busy', '1️⃣', 0), # Latin partial-word must NOT match (regression guard) ('I like to code in Python.', 'thon', 0), ('私はPythonでADKを使っています', 'thon', 0), From ed9250c5e7024fc22af9cf250019302fcb876f6c Mon Sep 17 00:00:00 2001 From: uo278036 Date: Mon, 16 Nov 2026 11:24:28 +0100 Subject: [PATCH 3/3] fix(memory): keep zero-width joiners inside words in memory search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both memory services split a word at a zero-width joiner (U+200D) or non-joiner (U+200C), because they are format characters and not marks. Bengali, Hindi and Persian use them inside words, so a search for the Bengali র‍্যাব became the tokens র and যাব and also returned unrelated text such as রাজ. Treat the joiners as word characters in both tokenizers. A joiner at the start or end of a word is dropped, like a leading mark, so the joiners of an emoji sequence are not words of their own. Fixes #8181 --- .../adk/memory/_sqlite_memory_service.py | 29 ++++++++++++---- .../adk/memory/in_memory_memory_service.py | 33 +++++++++++++----- .../memory/test_in_memory_memory_service.py | 13 +++++++ .../memory/test_sqlite_memory_service.py | 34 +++++++++++++++++++ 4 files changed, 95 insertions(+), 14 deletions(-) diff --git a/src/google/adk/memory/_sqlite_memory_service.py b/src/google/adk/memory/_sqlite_memory_service.py index 8214d80866d..3d94f6d44a9 100644 --- a/src/google/adk/memory/_sqlite_memory_service.py +++ b/src/google/adk/memory/_sqlite_memory_service.py @@ -17,6 +17,7 @@ import asyncio from collections.abc import AsyncIterator from collections.abc import Collection +from collections.abc import Iterable from collections.abc import Mapping from collections.abc import Sequence from contextlib import asynccontextmanager @@ -592,10 +593,28 @@ def _is_mark(c: str) -> bool: return unicodedata.category(c).startswith("M") -# Combining marks count as word characters: \w alone would split a word at -# them, such as the Thai vowel and tone marks in ข้าว. +# The zero-width non-joiner and joiner. Bengali, Hindi and Persian, among +# others, use them inside a word to pick a glyph form, such as the ZWJ +# between র and ্যাব in the Bengali word RAB. +_JOINERS = "\u200c\u200d" + + +# Combining marks and joiners count as word characters: \w alone would split +# a word at them, such as the Thai vowel and tone marks in ข้าว. def _is_token_char(c: str) -> bool: - return c.isalnum() or c in "_%" or _is_mark(c) + return c.isalnum() or c in "_%" or _is_mark(c) or c in _JOINERS + + +def _trim_token(chars: Iterable[str]) -> str: + """Joins chars, dropping leading marks and joiners and trailing joiners.""" + # Marks with no letter before them are dropped: the emoji variation + # selector U+FE0F is a mark, and as a token it would match any emoji. A + # joiner at the start or end of a token is dropped too, so the ZWJs that + # join the emoji of a family emoji are not tokens either. + token = "".join( + itertools.dropwhile(lambda c: _is_mark(c) or c in _JOINERS, chars) + ) + return token.rstrip(_JOINERS) def _extract_tokens(query: str) -> list[str]: @@ -606,9 +625,7 @@ def _extract_tokens(query: str) -> list[str]: for is_token, group in itertools.groupby(normalized, _is_token_char): if not is_token: continue - # Marks with no letter before them are dropped: the emoji variation - # selector U+FE0F is a mark, and as a token it would match any emoji. - lowered = "".join(itertools.dropwhile(_is_mark, group)).lower() + lowered = _trim_token(group).lower() if lowered and lowered not in seen: seen.add(lowered) tokens.append(lowered) diff --git a/src/google/adk/memory/in_memory_memory_service.py b/src/google/adk/memory/in_memory_memory_service.py index f25ec904e67..e797f290fa2 100644 --- a/src/google/adk/memory/in_memory_memory_service.py +++ b/src/google/adk/memory/in_memory_memory_service.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import annotations +from collections.abc import Iterable from collections.abc import Mapping from collections.abc import Sequence import itertools @@ -49,11 +50,29 @@ def _is_mark(c: str) -> bool: return unicodedata.category(c).startswith('M') -# Combining marks count as word characters: \w alone would split a word at -# them, such as the Thai vowel and tone marks in ข้าว. This matches the -# tokenizer of SqliteMemoryService. +# The zero-width non-joiner and joiner. Bengali, Hindi and Persian, among +# others, use them inside a word to pick a glyph form, such as the ZWJ +# between র and ্যাব in the Bengali word RAB. +_JOINERS = '\u200c\u200d' + + +# Combining marks and joiners count as word characters: \w alone would split +# a word at them, such as the Thai vowel and tone marks in ข้าว. This matches +# the tokenizer of SqliteMemoryService. def _is_word_char(c: str) -> bool: - return c.isalnum() or c == '_' or _is_mark(c) + return c.isalnum() or c == '_' or _is_mark(c) or c in _JOINERS + + +def _trim_word(chars: Iterable[str]) -> str: + """Joins chars, dropping leading marks and joiners and trailing joiners.""" + # Marks with no letter before them are dropped: the emoji variation + # selector U+FE0F is a mark, and as a word it would match any emoji. A + # joiner at the start or end of a word is dropped too, so the ZWJs that + # join the emoji of a family emoji are not words either. + word = ''.join( + itertools.dropwhile(lambda c: _is_mark(c) or c in _JOINERS, chars) + ) + return word.rstrip(_JOINERS) def _find_words(text: str) -> list[str]: @@ -62,9 +81,7 @@ def _find_words(text: str) -> list[str]: for is_word, group in itertools.groupby(text, _is_word_char): if not is_word: continue - # Marks with no letter before them are dropped: the emoji variation - # selector U+FE0F is a mark, and as a word it would match any emoji. - word = ''.join(itertools.dropwhile(_is_mark, group)) + word = _trim_word(group) if word: words.append(word) return words @@ -100,7 +117,7 @@ def _extract_searchable_words(text: str) -> set[str]: for _, group in itertools.groupby(word, _is_latin): # A combining mark on a Latin letter is not Latin itself, so it can # form a run of its own; like a leading mark, it is not a word. - run = ''.join(itertools.dropwhile(_is_mark, group)) + run = _trim_word(group) if run: words.add(run.lower()) return words diff --git a/tests/unittests/memory/test_in_memory_memory_service.py b/tests/unittests/memory/test_in_memory_memory_service.py index 5b390a94f1d..f6e081a2dc9 100644 --- a/tests/unittests/memory/test_in_memory_memory_service.py +++ b/tests/unittests/memory/test_in_memory_memory_service.py @@ -456,6 +456,19 @@ async def test_search_memory_does_not_collide_on_slash_in_identifiers(): # Keycap emoji: 1, U+FE0F and the enclosing keycap mark U+20E3 ('Room 1️⃣ is free', '1️⃣', 1), ('Room 2 is busy', '1️⃣', 0), + # A zero-width joiner or non-joiner inside a word does not split it + ('র\u200d্যাব এসেছে', 'র\u200d্যাব', 1), + ('আমার নাম রাজ', 'র\u200d্যাব', 0), + ('मैं क्\u200cष लिखता हूँ', 'क्\u200cष', 1), + ('मैं क्षमा माँगता हूँ', 'क्\u200cष', 0), + ('کتاب\u200cها روی میز است', 'کتاب\u200cها', 1), + ('مرد\u200cها آمدند', 'کتاب\u200cها', 0), + ('می\u200cخواهم بروم', 'خواهم', 1), + # A joiner at the start or end of a query word is not part of it + ('আমার নাম রাজ', 'রাজ\u200d', 1), + ('আমার নাম রাজ', '\u200cরাজ', 1), + # The joiners of an emoji sequence are not a word + ('I ❤️\u200d🔥 this', 'Family 👨\u200d👩\u200d👧', 0), # Latin partial-word must NOT match (regression guard) ('I like to code in Python.', 'thon', 0), ('私はPythonでADKを使っています', 'thon', 0), diff --git a/tests/unittests/memory/test_sqlite_memory_service.py b/tests/unittests/memory/test_sqlite_memory_service.py index 8251cebaa79..5bd7d6092ed 100644 --- a/tests/unittests/memory/test_sqlite_memory_service.py +++ b/tests/unittests/memory/test_sqlite_memory_service.py @@ -995,6 +995,40 @@ async def test_search_keeps_combining_marks_in_query_word( await service.close() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("text", "decoy", "query"), + [ + ("র\u200d্যাব এসেছে", "আমার নাম রাজ", "র\u200d্যাব"), + ("मैं क्\u200cष लिखता हूँ", "मैं क्षमा माँगता हूँ", "क्\u200cष"), + ("کتاب\u200cها روی میز است", "مرد\u200cها آمدند", "کتاب\u200cها"), + ("আমার নাম রাজ", "আমি ভাত খাই", "রাজ\u200d"), + ("আমার নাম রাজ", "আমি ভাত খাই", "\u200cরাজ"), + ( + "Family photo", + "I \u2764\ufe0f\u200d\U0001f525 this", + "photo \U0001f468\u200d\U0001f469\u200d\U0001f467", + ), + ], +) +async def test_search_keeps_joiners_in_query_word( + tmp_path, fts_mode, text, decoy, query +): + """A query word is not split at a zero-width joiner or non-joiner.""" + db_path = tmp_path / "memory.db" + service = SqliteMemoryService(db_path=db_path, fts=fts_mode) + session = _make_session([ + _make_event("user", text, 1.0), + _make_event("user", decoy, 2.0), + ]) + await service.add_session_to_memory(session) + response = await service.search_memory( + app_name="app", user_id="user", query=query + ) + assert [m.content.parts[0].text for m in response.memories] == [text] + await service.close() + + @pytest.mark.asyncio async def test_search_ignores_emoji_variation_selector(tmp_path, fts_mode): """The U+FE0F after an emoji in a query does not match other emoji."""