Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ This plugin uses embedded Turso (`@tursodatabase/database`) with `F32_BLOB` vect

### Upgrading from legacy SQLite shards

Startup recovers interrupted re-embed swaps, converts libSQL DiskANN indexes to the current Turso engine, and then verifies or upgrades the legacy shard schema. Engine conversion runs even when a store already has a completed legacy migration marker, and preserves stored vectors without re-embedding. Each converted database is backed up as `<database>.pre-tursodb-<timestamp>.bak`.

On first startup after upgrading, opencode-mem automatically migrates existing memory shard databases to native Turso/libSQL vector format:

- Each shard is backed up as `<shard>.db.legacy.bak` before rewrite
Expand Down
7 changes: 7 additions & 0 deletions src/services/turso/engine-migrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import {
withSqliteFileLockRetry,
renameSqliteDatabase,
copySqliteDatabase,
removeSqliteDatabase,
collectReleasedSqliteHandles,
} from "./sqlite-handle-release.js";
import { tursoConnectionManager, resolveDatabaseEncryption } from "./connection-manager.js";
import { TursoDb } from "./turso-db.js";
Expand Down Expand Up @@ -201,8 +203,10 @@ async function rewriteMemoryShard(dbPath: string): Promise<void> {
await tursoConnectionManager.closeConnection(stagedPath);
}

// Match encryption-migrator: Windows cannot rename over an existing file (EPERM).
await withSqliteFileLockRetry(() => {
copySqliteDatabase(dbPath, backupPath);
removeSqliteDatabase(dbPath);
renameSqliteDatabase(stagedPath, dbPath);
});

Expand Down Expand Up @@ -254,10 +258,13 @@ async function rewriteGenericDb(dbPath: string, tables: string[]): Promise<void>
}
} finally {
await staged.close();
await collectReleasedSqliteHandles();
}

// Match encryption-migrator: Windows cannot rename over an existing file (EPERM).
await withSqliteFileLockRetry(() => {
copySqliteDatabase(dbPath, backupPath);
removeSqliteDatabase(dbPath);
renameSqliteDatabase(stagedPath, dbPath);
});
log("Migrated auxiliary DB to @tursodatabase/database", { dbPath, backupPath });
Expand Down
6 changes: 6 additions & 0 deletions src/services/turso/legacy-migrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { CONFIG } from "../../config.js";
import { log } from "../logger.js";
import { tursoConnectionManager } from "./connection-manager.js";
import { tursoShardManager } from "./shard-manager.js";
import { runTursoEngineMigration } from "./engine-migrator.js";
import {
withSqliteFileLockRetry,
renameSqliteDatabase,
Expand Down Expand Up @@ -613,6 +614,11 @@ export async function runLegacyTursoMigration(): Promise<void> {

recoverInterruptedReembedSwaps();

// Recovery may put a libSQL shard back at its active path. Convert DiskANN
// indexes before any legacy verification opens it with the Turso engine,
// and before taking the legacy lock that would block the engine migration.
await runTursoEngineMigration();

const marker = readMarker(CONFIG.storagePath);
if (marker && (await allShardsComplete())) {
await reconcileShardRegistry();
Expand Down
2 changes: 0 additions & 2 deletions src/services/turso/ready.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { runLegacyTursoMigration } from "./legacy-migrator.js";
import { runTursoEngineMigration } from "./engine-migrator.js";
import { runDatabaseEncryptionMigration } from "./encryption-migrator.js";
import { tursoShardManager } from "./shard-manager.js";
import { log } from "../logger.js";
Expand All @@ -14,7 +13,6 @@ export async function ensureTursoReady(): Promise<void> {
initPromise = (async () => {
try {
await runLegacyTursoMigration();
await runTursoEngineMigration();
await runDatabaseEncryptionMigration();
const { shardPathMigrationService } = await import("../shard-path-migration-service.js");
await shardPathMigrationService.recoverInterruptedSwap();
Expand Down
209 changes: 203 additions & 6 deletions tests/turso-ready-gate.test.ts
Original file line number Diff line number Diff line change
@@ -1,32 +1,229 @@
import { describe, expect, it, afterEach } from "bun:test";
import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
import { createClient } from "@libsql/client";
import {
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
renameSync,
writeFileSync,
} from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { cleanupTursoTestDirectory } from "./turso-test-utils.js";

describe("turso ready gate", () => {
let baseDir: string;
let restoreConfig: (() => void) | undefined;

afterEach(async () => {
await cleanupTursoTestDirectory(baseDir);
restoreConfig?.();
restoreConfig = undefined;
});

it("runs legacy migration once and initializes metadata", async () => {
async function withStorage() {
baseDir = mkdtempSync(join(tmpdir(), "turso-ready-"));
const { CONFIG } = await import("../src/config.js");
const previous = {
storagePath: CONFIG.storagePath,
embeddingDimensions: CONFIG.embeddingDimensions,
databaseEncryptionEnabled: CONFIG.databaseEncryptionEnabled,
databaseEncryptionKey: CONFIG.databaseEncryptionKey,
};
restoreConfig = () => Object.assign(CONFIG, previous);
CONFIG.storagePath = baseDir;
CONFIG.embeddingDimensions = 768;
CONFIG.databaseEncryptionEnabled = false;
CONFIG.databaseEncryptionKey = undefined;
}

const scopeHash = "0123456789abcdef";
const vector = Array.from({ length: 1024 }, (_, i) => (i === 0 ? 1 : i === 1 ? 0.5 : 0));
const tagsVector = vector.map((value) => (value === 0 ? 0 : -value));

async function createIndexedShard(completedLegacyMigration: boolean): Promise<string> {
const projectsDir = join(baseDir, "projects");
mkdirSync(projectsDir);
const dbPath = join(projectsDir, `project_${scopeHash}_shard_0.db`);
const client = createClient({ url: `file:${dbPath}`, concurrency: 1 });
try {
await client.execute(`CREATE TABLE memories (
id TEXT PRIMARY KEY, content TEXT NOT NULL,
vector F32_BLOB(1024) NOT NULL, tags_vector F32_BLOB(1024),
container_tag TEXT NOT NULL, tags TEXT, type TEXT,
created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL,
metadata TEXT, display_name TEXT, user_name TEXT, user_email TEXT,
project_path TEXT, project_name TEXT, git_repo_url TEXT,
is_pinned INTEGER DEFAULT 0
)`);
for (const [id, tags] of [
["tagged", tagsVector],
["untagged", null],
] as const) {
await client.execute({
sql: `INSERT INTO memories VALUES (
?, ?, vector32(?), CASE WHEN ? IS NULL THEN NULL ELSE vector32(?) END,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)`,
args: [
id,
`synthetic ${id} memory`,
JSON.stringify(vector),
tags === null ? null : JSON.stringify(tags),
tags === null ? null : JSON.stringify(tags),
`opencode_project_${scopeHash}`,
tags === null ? null : '["migration"]',
"project",
123,
456,
'{"synthetic":true}',
"Test User",
"test-user",
"test@example.invalid",
"/synthetic/project",
"fixture",
"https://example.invalid/repo",
1,
],
});
}
await client.execute("CREATE TABLE shard_metadata (key TEXT PRIMARY KEY, value TEXT)");
await client.execute("INSERT INTO shard_metadata VALUES ('embedding_dimensions', '1024')");
await client.execute(
"INSERT INTO shard_metadata VALUES ('embedding_model', 'voyage-4-lite')"
);
for (const column of ["vector", "tags_vector"]) {
await client.execute(`CREATE INDEX memories_${column}_idx ON memories (
libsql_vector_idx(${column}, 'metric=cosine', 'compress_neighbors=float8', 'max_neighbors=20')
)`);
}
} finally {
client.close();
}
if (completedLegacyMigration) {
writeFileSync(
`${dbPath}.turso-migrate.json`,
JSON.stringify({
sourceCount: 2,
expectedCount: 2,
importedCount: 2,
skippedCount: 0,
status: "complete",
})
);
writeFileSync(
join(baseDir, ".turso-migrated"),
JSON.stringify({
completedAt: "2026-01-01T00:00:00Z",
shards: [{ path: dbPath, expectedCount: 2, importedCount: 2 }],
})
);
}
return dbPath;
}

async function assertMigratedShard(dbPath: string): Promise<void> {
const { tursoConnectionManager } = await import("../src/services/turso/connection-manager.js");
const db = await tursoConnectionManager.getConnection(dbPath);
const rows = await db.all(`SELECT *, vector_extract(vector) AS extracted,
CASE WHEN tags_vector IS NULL THEN NULL ELSE vector_extract(tags_vector) END AS extracted_tags
FROM memories ORDER BY id`);
expect(rows).toHaveLength(2);
for (const row of rows) {
expect(row.content).toBe(`synthetic ${row.id} memory`);
expect(JSON.parse(String(row.extracted))).toEqual(vector);
expect(row.container_tag).toBe(`opencode_project_${scopeHash}`);
expect(row.type).toBe("project");
expect(row.created_at).toBe(123);
expect(row.updated_at).toBe(456);
expect(row.metadata).toBe('{"synthetic":true}');
expect(row.display_name).toBe("Test User");
expect(row.user_name).toBe("test-user");
expect(row.user_email).toBe("test@example.invalid");
expect(row.project_path).toBe("/synthetic/project");
expect(row.project_name).toBe("fixture");
expect(row.git_repo_url).toBe("https://example.invalid/repo");
expect(row.is_pinned).toBe(1);
}
expect(rows[0]?.id).toBe("tagged");
expect(rows[0]?.tags).toBe('["migration"]');
expect(JSON.parse(String(rows[0]?.extracted_tags))).toEqual(tagsVector);
expect(rows[1]?.id).toBe("untagged");
expect(rows[1]?.tags).toBeNull();
expect(rows[1]?.tags_vector).toBeNull();
const metadata = await db.all("SELECT key, value FROM shard_metadata ORDER BY key");
expect(metadata).toEqual([
{ key: "embedding_dimensions", value: "1024" },
{ key: "embedding_model", value: "voyage-4-lite" },
]);
const { tursoShardManager } = await import("../src/services/turso/shard-manager.js");
const shards = await tursoShardManager.getAllShards("project", "");
expect(shards).toHaveLength(1);
expect(shards[0]?.scopeHash).toBe(scopeHash);
expect(shards[0]?.vectorCount).toBe(2);
expect(shards[0]?.isActive).toBe(true);
expect(existsSync(join(baseDir, ".tursodb-engine-v1"))).toBe(true);
expect(existsSync(join(baseDir, ".turso-migrated"))).toBe(true);
}

it("runs legacy migration once and initializes metadata", async () => {
await withStorage();
const { CONFIG } = await import("../src/config.js");
CONFIG.storagePath = join(baseDir, "new-store");

const { ensureTursoReady } = await import("../src/services/turso/ready.js");
await ensureTursoReady();
await ensureTursoReady();

expect(existsSync(join(baseDir, "metadata.db"))).toBe(true);
expect(existsSync(join(CONFIG.storagePath, "metadata.db"))).toBe(true);
});

for (const completedLegacyMigration of [false, true]) {
it(`migrates indexed libSQL shards through startup (legacy marker: ${completedLegacyMigration})`, async () => {
await withStorage();
const dbPath = await createIndexedShard(completedLegacyMigration);
const { ensureTursoReady } = await import("../src/services/turso/ready.js");
await ensureTursoReady();
await assertMigratedShard(dbPath);
const backups = () =>
readdirSync(join(baseDir, "projects")).filter(
(name) => name.includes(".pre-tursodb-") && name.endsWith(".bak")
);
expect(backups()).toHaveLength(1);
const marker = readFileSync(join(baseDir, ".tursodb-engine-v1"), "utf-8");
const { closeTursoAndInvalidateCaches } = await import("../src/services/turso/lifecycle.js");
await closeTursoAndInvalidateCaches();
await ensureTursoReady();
await assertMigratedShard(dbPath);
expect(backups()).toHaveLength(1);
expect(readFileSync(join(baseDir, ".tursodb-engine-v1"), "utf-8")).toBe(marker);
}, 15000);
}

it("recovers an interrupted re-embed swap before converting its libSQL indexes", async () => {
await withStorage();
const dbPath = await createIndexedShard(true);
const stagedPath = `${dbPath}.reembed-fixture.tmp`;
renameSync(dbPath, stagedPath);
writeFileSync(
`${dbPath}.reembed-swap.json`,
JSON.stringify({
dbPath,
stagedPath,
backupPath: `${dbPath}.pre-reembed-fixture.bak`,
})
);
const { ensureTursoReady } = await import("../src/services/turso/ready.js");
await ensureTursoReady();
await assertMigratedShard(dbPath);
expect(existsSync(stagedPath)).toBe(false);
expect(existsSync(`${dbPath}.reembed-swap.json`)).toBe(false);
}, 15000);

it("throws when migration lock is held by another live process", async () => {
baseDir = mkdtempSync(join(tmpdir(), "turso-ready-lock-"));
const { CONFIG } = await import("../src/config.js");
CONFIG.storagePath = baseDir;
await withStorage();

writeFileSync(
join(baseDir, ".turso-migrate.lock"),
Expand Down