Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 23 additions & 6 deletions src/google/adk/memory/_sqlite_memory_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import asyncio
from collections.abc import AsyncIterator
from collections.abc import Collection
from collections.abc import Iterable
from collections.abc import Mapping
from collections.abc import Sequence
from contextlib import asynccontextmanager
Expand Down Expand Up @@ -592,10 +593,28 @@ def _is_mark(c: str) -> bool:
return unicodedata.category(c).startswith("M")


# Combining marks count as word characters: \w alone would split a word at
# them, such as the Thai vowel and tone marks in ข้าว.
# The zero-width non-joiner and joiner. Bengali, Hindi and Persian, among
# others, use them inside a word to pick a glyph form, such as the ZWJ
# between র and ্যাব in the Bengali word RAB.
_JOINERS = "\u200c\u200d"


# Combining marks and joiners count as word characters: \w alone would split
# a word at them, such as the Thai vowel and tone marks in ข้าว.
def _is_token_char(c: str) -> bool:
return c.isalnum() or c in "_%" or _is_mark(c)
return c.isalnum() or c in "_%" or _is_mark(c) or c in _JOINERS


def _trim_token(chars: Iterable[str]) -> str:
"""Joins chars, dropping leading marks and joiners and trailing joiners."""
# Marks with no letter before them are dropped: the emoji variation
# selector U+FE0F is a mark, and as a token it would match any emoji. A
# joiner at the start or end of a token is dropped too, so the ZWJs that
# join the emoji of a family emoji are not tokens either.
token = "".join(
itertools.dropwhile(lambda c: _is_mark(c) or c in _JOINERS, chars)
)
return token.rstrip(_JOINERS)


def _extract_tokens(query: str) -> list[str]:
Expand All @@ -606,9 +625,7 @@ def _extract_tokens(query: str) -> list[str]:
for is_token, group in itertools.groupby(normalized, _is_token_char):
if not is_token:
continue
# Marks with no letter before them are dropped: the emoji variation
# selector U+FE0F is a mark, and as a token it would match any emoji.
lowered = "".join(itertools.dropwhile(_is_mark, group)).lower()
lowered = _trim_token(group).lower()
if lowered and lowered not in seen:
seen.add(lowered)
tokens.append(lowered)
Expand Down
59 changes: 51 additions & 8 deletions src/google/adk/memory/in_memory_memory_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@
# limitations under the License.
from __future__ import annotations

from collections.abc import Iterable
from collections.abc import Mapping
from collections.abc import Sequence
import itertools
import re
import threading
from typing import TYPE_CHECKING
import unicodedata
Expand Down Expand Up @@ -46,18 +46,59 @@ def _is_latin(c: str) -> bool:
return unicodedata.name(c, '').startswith('LATIN')


def _is_mark(c: str) -> bool:
return unicodedata.category(c).startswith('M')


# The zero-width non-joiner and joiner. Bengali, Hindi and Persian, among
# others, use them inside a word to pick a glyph form, such as the ZWJ
# between র and ্যাব in the Bengali word RAB.
_JOINERS = '\u200c\u200d'


# Combining marks and joiners count as word characters: \w alone would split
# a word at them, such as the Thai vowel and tone marks in ข้าว. This matches
# the tokenizer of SqliteMemoryService.
def _is_word_char(c: str) -> bool:
return c.isalnum() or c == '_' or _is_mark(c) or c in _JOINERS


def _trim_word(chars: Iterable[str]) -> str:
"""Joins chars, dropping leading marks and joiners and trailing joiners."""
# Marks with no letter before them are dropped: the emoji variation
# selector U+FE0F is a mark, and as a word it would match any emoji. A
# joiner at the start or end of a word is dropped too, so the ZWJs that
# join the emoji of a family emoji are not words either.
word = ''.join(
itertools.dropwhile(lambda c: _is_mark(c) or c in _JOINERS, chars)
)
return word.rstrip(_JOINERS)


def _find_words(text: str) -> list[str]:
"""Returns the words of an NFC-normalized text, marks included."""
words = []
for is_word, group in itertools.groupby(text, _is_word_char):
if not is_word:
continue
word = _trim_word(group)
if word:
words.append(word)
return words


def _extract_words_lower(text: str) -> set[str]:
"""Extracts Unicode-aware tokens from a string in lowercase."""
text = unicodedata.normalize('NFC', text)
return set(word.lower() for word in re.findall(r'\w+', text))
return set(word.lower() for word in _find_words(text))


def _extract_searchable_words(text: str) -> set[str]:
r"""Extracts the words a query or an event is matched on, in lowercase.

The tokens of _extract_words_lower, plus, for a token that mixes Latin and
non-Latin scripts, each of its single-script runs. Japanese and Chinese are
written without spaces, so 私はPythonを使う is a single \w+ token and a query
written without spaces, so 私はPythonを使う is a single token and a query
for Python matches nothing. Splitting on the boundary between Latin and
non-Latin scripts makes the embedded Latin word a token of its own, while
still keeping a partial word such as thon from matching. Queries and events
Expand All @@ -68,15 +109,17 @@ def _extract_searchable_words(text: str) -> set[str]:
text: The text of a query or an event.
"""
words: set[str] = set()
for word in re.findall(r'\w+', unicodedata.normalize('NFC', text)):
for word in _find_words(unicodedata.normalize('NFC', text)):
words.add(word.lower())
if not word.isascii():
# Split before lowercasing: "İ".lower() is "i" plus U+0307, which is
# not Latin, so splitting afterwards yields a stray "i" word.
words.update(
''.join(group).lower()
for _, group in itertools.groupby(word, _is_latin)
)
for _, group in itertools.groupby(word, _is_latin):
# A combining mark on a Latin letter is not Latin itself, so it can
# form a run of its own; like a leading mark, it is not a word.
run = _trim_word(group)
if run:
words.add(run.lower())
return words


Expand Down
34 changes: 34 additions & 0 deletions tests/unittests/memory/test_in_memory_memory_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,40 @@ async def test_search_memory_does_not_collide_on_slash_in_identifiers():
('私と犬', 'Pythonと', 0),
# "İ".lower() is "i" plus U+0307: no stray "i" word to match on
('I like cats.', 'İstanbul旅行', 0),
# Combining marks are part of a word: Thai vowel and tone marks,
# Devanagari vowel signs
('ข้าวผัดอร่อยมาก', 'ข้าว', 1),
('ขอบคุณครับ', 'ข้าว', 0),
('मुझे हिंदी पसंद है', 'हिंदी', 1),
('मैं हैदराबाद में रहता हूँ', 'हिंदी', 0),
# A mark with no letter before it, such as the emoji variation
# selector U+FE0F, is not a word
('I love cats ❤️', 'Dogs ❤️', 0),
# Bengali and Tamil vowel signs and virama
('আমি ভাত খাই', 'ভাত', 1),
('ভালো আছি', 'ভাত', 0),
('தமிழ் எனக்கு பிடிக்கும்', 'தமிழ்', 1),
('நான் சாதம் சாப்பிடுகிறேன்', 'தமிழ்', 0),
# Latin word inside Thai text
('ฉันใช้Pythonเขียนโปรแกรม', 'Python', 1),
('ฉันใช้Pythonเขียนโปรแกรม', 'เขียน', 1),
('ฉันใช้Pythonเขียนโปรแกรม', 'ข้าว', 0),
# Keycap emoji: 1, U+FE0F and the enclosing keycap mark U+20E3
('Room 1️⃣ is free', '1️⃣', 1),
('Room 2 is busy', '1️⃣', 0),
# A zero-width joiner or non-joiner inside a word does not split it
('র\u200d্যাব এসেছে', 'র\u200d্যাব', 1),
('আমার নাম রাজ', 'র\u200d্যাব', 0),
('मैं क्\u200cष लिखता हूँ', 'क्\u200cष', 1),
('मैं क्षमा माँगता हूँ', 'क्\u200cष', 0),
('کتاب\u200cها روی میز است', 'کتاب\u200cها', 1),
('مرد\u200cها آمدند', 'کتاب\u200cها', 0),
('می\u200cخواهم بروم', 'خواهم', 1),
# A joiner at the start or end of a query word is not part of it
('আমার নাম রাজ', 'রাজ\u200d', 1),
('আমার নাম রাজ', '\u200cরাজ', 1),
# The joiners of an emoji sequence are not a word
('I ❤️\u200d🔥 this', 'Family 👨\u200d👩\u200d👧', 0),
# Latin partial-word must NOT match (regression guard)
('I like to code in Python.', 'thon', 0),
('私はPythonでADKを使っています', 'thon', 0),
Expand Down
34 changes: 34 additions & 0 deletions tests/unittests/memory/test_sqlite_memory_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,40 @@ async def test_search_keeps_combining_marks_in_query_word(
await service.close()


@pytest.mark.asyncio
@pytest.mark.parametrize(
("text", "decoy", "query"),
[
("র\u200d্যাব এসেছে", "আমার নাম রাজ", "র\u200d্যাব"),
("मैं क्\u200cष लिखता हूँ", "मैं क्षमा माँगता हूँ", "क्\u200cष"),
("کتاب\u200cها روی میز است", "مرد\u200cها آمدند", "کتاب\u200cها"),
("আমার নাম রাজ", "আমি ভাত খাই", "রাজ\u200d"),
("আমার নাম রাজ", "আমি ভাত খাই", "\u200cরাজ"),
(
"Family photo",
"I \u2764\ufe0f\u200d\U0001f525 this",
"photo \U0001f468\u200d\U0001f469\u200d\U0001f467",
),
],
)
async def test_search_keeps_joiners_in_query_word(
tmp_path, fts_mode, text, decoy, query
):
"""A query word is not split at a zero-width joiner or non-joiner."""
db_path = tmp_path / "memory.db"
service = SqliteMemoryService(db_path=db_path, fts=fts_mode)
session = _make_session([
_make_event("user", text, 1.0),
_make_event("user", decoy, 2.0),
])
await service.add_session_to_memory(session)
response = await service.search_memory(
app_name="app", user_id="user", query=query
)
assert [m.content.parts[0].text for m in response.memories] == [text]
await service.close()


@pytest.mark.asyncio
async def test_search_ignores_emoji_variation_selector(tmp_path, fts_mode):
"""The U+FE0F after an emoji in a query does not match other emoji."""
Expand Down
Loading