diff --git a/src/google/adk/memory/_sqlite_memory_service.py b/src/google/adk/memory/_sqlite_memory_service.py index 8214d80866d..3d94f6d44a9 100644 --- a/src/google/adk/memory/_sqlite_memory_service.py +++ b/src/google/adk/memory/_sqlite_memory_service.py @@ -17,6 +17,7 @@ import asyncio from collections.abc import AsyncIterator from collections.abc import Collection +from collections.abc import Iterable from collections.abc import Mapping from collections.abc import Sequence from contextlib import asynccontextmanager @@ -592,10 +593,28 @@ def _is_mark(c: str) -> bool: return unicodedata.category(c).startswith("M") -# Combining marks count as word characters: \w alone would split a word at -# them, such as the Thai vowel and tone marks in ข้าว. +# The zero-width non-joiner and joiner. Bengali, Hindi and Persian, among +# others, use them inside a word to pick a glyph form, such as the ZWJ +# between র and ্যাব in the Bengali word RAB. +_JOINERS = "\u200c\u200d" + + +# Combining marks and joiners count as word characters: \w alone would split +# a word at them, such as the Thai vowel and tone marks in ข้าว. def _is_token_char(c: str) -> bool: - return c.isalnum() or c in "_%" or _is_mark(c) + return c.isalnum() or c in "_%" or _is_mark(c) or c in _JOINERS + + +def _trim_token(chars: Iterable[str]) -> str: + """Joins chars, dropping leading marks and joiners and trailing joiners.""" + # Marks with no letter before them are dropped: the emoji variation + # selector U+FE0F is a mark, and as a token it would match any emoji. A + # joiner at the start or end of a token is dropped too, so the ZWJs that + # join the emoji of a family emoji are not tokens either. + token = "".join( + itertools.dropwhile(lambda c: _is_mark(c) or c in _JOINERS, chars) + ) + return token.rstrip(_JOINERS) def _extract_tokens(query: str) -> list[str]: @@ -606,9 +625,7 @@ def _extract_tokens(query: str) -> list[str]: for is_token, group in itertools.groupby(normalized, _is_token_char): if not is_token: continue - # Marks with no letter before them are dropped: the emoji variation - # selector U+FE0F is a mark, and as a token it would match any emoji. - lowered = "".join(itertools.dropwhile(_is_mark, group)).lower() + lowered = _trim_token(group).lower() if lowered and lowered not in seen: seen.add(lowered) tokens.append(lowered) diff --git a/src/google/adk/memory/in_memory_memory_service.py b/src/google/adk/memory/in_memory_memory_service.py index 11c974165bd..e797f290fa2 100644 --- a/src/google/adk/memory/in_memory_memory_service.py +++ b/src/google/adk/memory/in_memory_memory_service.py @@ -13,10 +13,10 @@ # limitations under the License. from __future__ import annotations +from collections.abc import Iterable from collections.abc import Mapping from collections.abc import Sequence import itertools -import re import threading from typing import TYPE_CHECKING import unicodedata @@ -46,10 +46,51 @@ def _is_latin(c: str) -> bool: return unicodedata.name(c, '').startswith('LATIN') +def _is_mark(c: str) -> bool: + return unicodedata.category(c).startswith('M') + + +# The zero-width non-joiner and joiner. Bengali, Hindi and Persian, among +# others, use them inside a word to pick a glyph form, such as the ZWJ +# between র and ্যাব in the Bengali word RAB. +_JOINERS = '\u200c\u200d' + + +# Combining marks and joiners count as word characters: \w alone would split +# a word at them, such as the Thai vowel and tone marks in ข้าว. This matches +# the tokenizer of SqliteMemoryService. +def _is_word_char(c: str) -> bool: + return c.isalnum() or c == '_' or _is_mark(c) or c in _JOINERS + + +def _trim_word(chars: Iterable[str]) -> str: + """Joins chars, dropping leading marks and joiners and trailing joiners.""" + # Marks with no letter before them are dropped: the emoji variation + # selector U+FE0F is a mark, and as a word it would match any emoji. A + # joiner at the start or end of a word is dropped too, so the ZWJs that + # join the emoji of a family emoji are not words either. + word = ''.join( + itertools.dropwhile(lambda c: _is_mark(c) or c in _JOINERS, chars) + ) + return word.rstrip(_JOINERS) + + +def _find_words(text: str) -> list[str]: + """Returns the words of an NFC-normalized text, marks included.""" + words = [] + for is_word, group in itertools.groupby(text, _is_word_char): + if not is_word: + continue + word = _trim_word(group) + if word: + words.append(word) + return words + + def _extract_words_lower(text: str) -> set[str]: """Extracts Unicode-aware tokens from a string in lowercase.""" text = unicodedata.normalize('NFC', text) - return set(word.lower() for word in re.findall(r'\w+', text)) + return set(word.lower() for word in _find_words(text)) def _extract_searchable_words(text: str) -> set[str]: @@ -57,7 +98,7 @@ def _extract_searchable_words(text: str) -> set[str]: The tokens of _extract_words_lower, plus, for a token that mixes Latin and non-Latin scripts, each of its single-script runs. Japanese and Chinese are - written without spaces, so 私はPythonを使う is a single \w+ token and a query + written without spaces, so 私はPythonを使う is a single token and a query for Python matches nothing. Splitting on the boundary between Latin and non-Latin scripts makes the embedded Latin word a token of its own, while still keeping a partial word such as thon from matching. Queries and events @@ -68,15 +109,17 @@ def _extract_searchable_words(text: str) -> set[str]: text: The text of a query or an event. """ words: set[str] = set() - for word in re.findall(r'\w+', unicodedata.normalize('NFC', text)): + for word in _find_words(unicodedata.normalize('NFC', text)): words.add(word.lower()) if not word.isascii(): # Split before lowercasing: "İ".lower() is "i" plus U+0307, which is # not Latin, so splitting afterwards yields a stray "i" word. - words.update( - ''.join(group).lower() - for _, group in itertools.groupby(word, _is_latin) - ) + for _, group in itertools.groupby(word, _is_latin): + # A combining mark on a Latin letter is not Latin itself, so it can + # form a run of its own; like a leading mark, it is not a word. + run = _trim_word(group) + if run: + words.add(run.lower()) return words diff --git a/tests/unittests/memory/test_in_memory_memory_service.py b/tests/unittests/memory/test_in_memory_memory_service.py index 645385429b9..f6e081a2dc9 100644 --- a/tests/unittests/memory/test_in_memory_memory_service.py +++ b/tests/unittests/memory/test_in_memory_memory_service.py @@ -435,6 +435,40 @@ async def test_search_memory_does_not_collide_on_slash_in_identifiers(): ('私と犬', 'Pythonと', 0), # "İ".lower() is "i" plus U+0307: no stray "i" word to match on ('I like cats.', 'İstanbul旅行', 0), + # Combining marks are part of a word: Thai vowel and tone marks, + # Devanagari vowel signs + ('ข้าวผัดอร่อยมาก', 'ข้าว', 1), + ('ขอบคุณครับ', 'ข้าว', 0), + ('मुझे हिंदी पसंद है', 'हिंदी', 1), + ('मैं हैदराबाद में रहता हूँ', 'हिंदी', 0), + # A mark with no letter before it, such as the emoji variation + # selector U+FE0F, is not a word + ('I love cats ❤️', 'Dogs ❤️', 0), + # Bengali and Tamil vowel signs and virama + ('আমি ভাত খাই', 'ভাত', 1), + ('ভালো আছি', 'ভাত', 0), + ('தமிழ் எனக்கு பிடிக்கும்', 'தமிழ்', 1), + ('நான் சாதம் சாப்பிடுகிறேன்', 'தமிழ்', 0), + # Latin word inside Thai text + ('ฉันใช้Pythonเขียนโปรแกรม', 'Python', 1), + ('ฉันใช้Pythonเขียนโปรแกรม', 'เขียน', 1), + ('ฉันใช้Pythonเขียนโปรแกรม', 'ข้าว', 0), + # Keycap emoji: 1, U+FE0F and the enclosing keycap mark U+20E3 + ('Room 1️⃣ is free', '1️⃣', 1), + ('Room 2 is busy', '1️⃣', 0), + # A zero-width joiner or non-joiner inside a word does not split it + ('র\u200d্যাব এসেছে', 'র\u200d্যাব', 1), + ('আমার নাম রাজ', 'র\u200d্যাব', 0), + ('मैं क्\u200cष लिखता हूँ', 'क्\u200cष', 1), + ('मैं क्षमा माँगता हूँ', 'क्\u200cष', 0), + ('کتاب\u200cها روی میز است', 'کتاب\u200cها', 1), + ('مرد\u200cها آمدند', 'کتاب\u200cها', 0), + ('می\u200cخواهم بروم', 'خواهم', 1), + # A joiner at the start or end of a query word is not part of it + ('আমার নাম রাজ', 'রাজ\u200d', 1), + ('আমার নাম রাজ', '\u200cরাজ', 1), + # The joiners of an emoji sequence are not a word + ('I ❤️\u200d🔥 this', 'Family 👨\u200d👩\u200d👧', 0), # Latin partial-word must NOT match (regression guard) ('I like to code in Python.', 'thon', 0), ('私はPythonでADKを使っています', 'thon', 0), diff --git a/tests/unittests/memory/test_sqlite_memory_service.py b/tests/unittests/memory/test_sqlite_memory_service.py index 8251cebaa79..5bd7d6092ed 100644 --- a/tests/unittests/memory/test_sqlite_memory_service.py +++ b/tests/unittests/memory/test_sqlite_memory_service.py @@ -995,6 +995,40 @@ async def test_search_keeps_combining_marks_in_query_word( await service.close() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("text", "decoy", "query"), + [ + ("র\u200d্যাব এসেছে", "আমার নাম রাজ", "র\u200d্যাব"), + ("मैं क्\u200cष लिखता हूँ", "मैं क्षमा माँगता हूँ", "क्\u200cष"), + ("کتاب\u200cها روی میز است", "مرد\u200cها آمدند", "کتاب\u200cها"), + ("আমার নাম রাজ", "আমি ভাত খাই", "রাজ\u200d"), + ("আমার নাম রাজ", "আমি ভাত খাই", "\u200cরাজ"), + ( + "Family photo", + "I \u2764\ufe0f\u200d\U0001f525 this", + "photo \U0001f468\u200d\U0001f469\u200d\U0001f467", + ), + ], +) +async def test_search_keeps_joiners_in_query_word( + tmp_path, fts_mode, text, decoy, query +): + """A query word is not split at a zero-width joiner or non-joiner.""" + db_path = tmp_path / "memory.db" + service = SqliteMemoryService(db_path=db_path, fts=fts_mode) + session = _make_session([ + _make_event("user", text, 1.0), + _make_event("user", decoy, 2.0), + ]) + await service.add_session_to_memory(session) + response = await service.search_memory( + app_name="app", user_id="user", query=query + ) + assert [m.content.parts[0].text for m in response.memories] == [text] + await service.close() + + @pytest.mark.asyncio async def test_search_ignores_emoji_variation_selector(tmp_path, fts_mode): """The U+FE0F after an emoji in a query does not match other emoji."""