Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ This plugin uses embedded Turso (`@tursodatabase/database`) with `F32_BLOB` vect

Startup recovers interrupted re-embed swaps, converts libSQL DiskANN indexes to the current Turso engine, and then verifies or upgrades the legacy shard schema. Engine conversion runs even when a store already has a completed legacy migration marker, and preserves stored vectors without re-embedding. Each converted database is backed up as `<database>.pre-tursodb-<timestamp>.bak`.

On macOS and Linux, multiple OpenCode sessions can share the same `storagePath` via Turso’s experimental `multiprocess_wal` (every process must use the same mode — restart all sessions after upgrading). On Windows the engine rejects that flag, so only one OpenCode session can own the memory databases at a time.

On first startup after upgrading, opencode-mem automatically migrates existing memory shard databases to native Turso/libSQL vector format:

- Each shard is backed up as `<shard>.db.legacy.bak` before rewrite
Expand Down
9 changes: 8 additions & 1 deletion src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -324,11 +324,18 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => {
} catch (error) {
log("Turso ready gate failed before web server start", { error: String(error) });
if (ctx.client?.tui) {
const { isTursoMultiProcessLockError } =
await import("./services/turso/connection-manager.js");
const lockHeld = isTursoMultiProcessLockError(error);
ctx.client.tui
.showToast({
body: {
title: "Memory Explorer",
message: "Database migration failed; web UI not started",
message: lockHeld
? process.platform === "win32"
? "Memory DB locked by another OpenCode session (Windows is single-owner)"
: "Memory DB locked by another session — close it or restart all OpenCode windows"
: "Database migration failed; web UI not started",
variant: "error",
duration: 8000,
},
Expand Down
59 changes: 56 additions & 3 deletions src/services/turso/connection-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,29 @@ import { resolveOrCreateDatabaseEncryptionKey } from "./encryption-key.js";

export type ConnectFactory = (path: string, opts?: DatabaseOpts) => Promise<Database>;

/** Always-on experimental flags for every Turso open. */
export const TURSO_BASE_EXPERIMENTAL_FEATURES = ["encryption"] as const;

/**
* Multiprocess WAL is Unix-only. On Windows the default IO backend rejects the
* flag (`experimental multiprocess WAL is not supported by the active IO backend`).
*/
export function supportsTursoMultiprocessWal(
platform: NodeJS.Platform = process.platform
): boolean {
return platform !== "win32";
}

/** Experimental flags required on every Turso open for the current platform. */
export function tursoExperimentalFeatures(
platform: NodeJS.Platform = process.platform
): Array<(typeof TURSO_BASE_EXPERIMENTAL_FEATURES)[number] | "multiprocess_wal"> {
if (supportsTursoMultiprocessWal(platform)) {
return [...TURSO_BASE_EXPERIMENTAL_FEATURES, "multiprocess_wal"];
}
return [...TURSO_BASE_EXPERIMENTAL_FEATURES];
}

function assertPathInsideStorage(dbPath: string): void {
const storageRoot = resolve(CONFIG.storagePath);
const resolvedPath = resolve(dbPath);
Expand All @@ -20,9 +43,14 @@ function assertPathInsideStorage(dbPath: string): void {
}
}

function buildConnectOptions(encryption?: EncryptionOpts | null): DatabaseOpts {
/**
* Shared connect options for `@tursodatabase/database`.
* Always enables encryption; adds multiprocess_wal on Unix so concurrent
* OpenCode sessions can share the store. Windows stays single-process.
*/
export function buildConnectOptions(encryption?: EncryptionOpts | null): DatabaseOpts {
const opts: DatabaseOpts = {
experimental: ["encryption"],
experimental: tursoExperimentalFeatures(),
};
if (encryption) {
opts.encryption = encryption;
Expand All @@ -39,6 +67,28 @@ export function resolveDatabaseEncryption(): EncryptionOpts | null {
};
}

const LOCK_ERROR_RE =
/File is locked by another process|already open (with|without) experimental multiprocess WAL|Locking error/i;

export function isTursoMultiProcessLockError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return LOCK_ERROR_RE.test(message);
}

export function wrapTursoOpenError(dbPath: string, error: unknown): Error {
const message = error instanceof Error ? error.message : String(error);
if (isTursoMultiProcessLockError(error)) {
const platformHint =
process.platform === "win32"
? "On Windows the Turso engine still allows only one process as database owner — close other OpenCode sessions."
: "Another OpenCode session (or an older opencode-mem without multiprocess_wal) still holds the database — close it and retry.";
return new Error(`Failed to open database ${dbPath}: ${message}. ${platformHint}`, {
cause: error,
});
}
return error instanceof Error ? error : new Error(message, { cause: error });
}

export class TursoConnectionManager {
private readonly connections = new Map<string, TursoDb>();
private readonly pending = new Map<string, Promise<TursoDb>>();
Expand Down Expand Up @@ -92,13 +142,16 @@ export class TursoConnectionManager {
}
if (encryption) {
const message = error instanceof Error ? error.message : String(error);
if (isTursoMultiProcessLockError(error)) {
throw wrapTursoOpenError(dbPath, error);
}
throw new Error(
`Failed to open encrypted database ${dbPath}: ${message}. ` +
`Check databaseEncryptionKey / cipher, or remove encryption config for plaintext shards.`,
{ cause: error }
);
}
throw error;
throw wrapTursoOpenError(dbPath, error);
}
})();

Expand Down
22 changes: 10 additions & 12 deletions src/services/turso/encryption-migrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@ import { existsSync, readdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { CONFIG } from "../../config.js";
import { log } from "../logger.js";
import { resolveDatabaseEncryption, tursoConnectionManager } from "./connection-manager.js";
import {
resolveDatabaseEncryption,
tursoConnectionManager,
buildConnectOptions,
} from "./connection-manager.js";
import { acquireTursoOperationLock } from "./operation-lock.js";
import {
withSqliteFileLockRetry,
Expand Down Expand Up @@ -51,10 +55,7 @@ async function isAlreadyEncrypted(dbPath: string): Promise<boolean> {
const encryption = resolveDatabaseEncryption();
if (!encryption) return false;
try {
const db = await connect(dbPath, {
encryption,
experimental: ["encryption"],
});
const db = await connect(dbPath, buildConnectOptions(encryption));
await db.close();
return true;
} catch {
Expand All @@ -64,7 +65,7 @@ async function isAlreadyEncrypted(dbPath: string): Promise<boolean> {

async function canOpenPlain(dbPath: string): Promise<boolean> {
try {
const db = await connect(dbPath, { experimental: ["encryption"] });
const db = await connect(dbPath, buildConnectOptions());
await db.close();
return true;
} catch {
Expand All @@ -84,12 +85,9 @@ async function dumpAndReloadEncrypted(dbPath: string): Promise<void> {
const backupPath = `${dbPath}.pre-encrypt-${Date.now()}.bak`;

// Re-open plain to copy rows into encrypted staged DB.
const source = await connect(dbPath, { experimental: ["encryption"] });
const source = await connect(dbPath, buildConnectOptions());
const sourceDb = new TursoDb(source);
const staged = await connect(stagedPath, {
encryption,
experimental: ["encryption"],
});
const staged = await connect(stagedPath, buildConnectOptions(encryption));
const stagedDb = new TursoDb(staged);

try {
Expand Down Expand Up @@ -214,7 +212,7 @@ async function dumpAndReloadEncrypted(dbPath: string): Promise<void> {
});

// Verify encrypted open
const verify = await connect(dbPath, { encryption, experimental: ["encryption"] });
const verify = await connect(dbPath, buildConnectOptions(encryption));
await verify.close();

log("Encrypted database at rest", { dbPath, backupPath });
Expand Down
16 changes: 7 additions & 9 deletions src/services/turso/engine-migrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ import {
removeSqliteDatabase,
collectReleasedSqliteHandles,
} from "./sqlite-handle-release.js";
import { tursoConnectionManager, resolveDatabaseEncryption } from "./connection-manager.js";
import {
tursoConnectionManager,
resolveDatabaseEncryption,
buildConnectOptions,
} from "./connection-manager.js";
import { TursoDb } from "./turso-db.js";
import { tursoShardManager } from "./shard-manager.js";
import { parseExtractedVector } from "./vector-utils.js";
Expand Down Expand Up @@ -52,10 +56,7 @@ function collectCandidateDbs(): string[] {
async function needsEngineRewrite(dbPath: string): Promise<boolean> {
try {
const encryption = resolveDatabaseEncryption();
const opts = encryption
? { encryption, experimental: ["encryption" as const] }
: { experimental: ["encryption" as const] };
const db = await connect(dbPath, opts);
const db = await connect(dbPath, buildConnectOptions(encryption));
try {
await db.all("SELECT 1");
return false;
Expand Down Expand Up @@ -222,10 +223,7 @@ async function rewriteGenericDb(dbPath: string, tables: string[]): Promise<void>

await tursoConnectionManager.closeConnection(dbPath);
const encryption = resolveDatabaseEncryption();
const opts = encryption
? { encryption, experimental: ["encryption" as const] }
: { experimental: ["encryption" as const] };
const stagedNative = await connect(stagedPath, opts);
const stagedNative = await connect(stagedPath, buildConnectOptions(encryption));
const staged = new TursoDb(stagedNative);
try {
// Recreate schema by copying CREATE SQL from source via libsql.
Expand Down
20 changes: 15 additions & 5 deletions src/services/turso/legacy-migrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ async function allShardsComplete(): Promise<boolean> {
return true;
}

function recoverInterruptedReembedSwaps(): void {
async function recoverInterruptedReembedSwaps(): Promise<void> {
for (const dirName of ["users", "projects"] as const) {
const dir = join(CONFIG.storagePath, dirName);
if (!existsSync(dir)) continue;
Expand Down Expand Up @@ -543,11 +543,21 @@ function recoverInterruptedReembedSwaps(): void {
}

if (existsSync(expectedDbPath)) {
if (existsSync(stagedPath)) removeSqliteDatabase(stagedPath);
if (existsSync(stagedPath)) {
await withSqliteFileLockRetry(() => removeSqliteDatabase(stagedPath));
}
} else if (existsSync(stagedPath)) {
renameSqliteDatabase(stagedPath, expectedDbPath);
// Target is absent after a crash mid-swap; retry covers Windows EPERM
// while native handles release (same pattern as engine/encryption swaps).
await withSqliteFileLockRetry(() => {
if (existsSync(expectedDbPath)) removeSqliteDatabase(expectedDbPath);
renameSqliteDatabase(stagedPath, expectedDbPath);
});
} else if (existsSync(backupPath)) {
renameSqliteDatabase(backupPath, expectedDbPath);
await withSqliteFileLockRetry(() => {
if (existsSync(expectedDbPath)) removeSqliteDatabase(expectedDbPath);
renameSqliteDatabase(backupPath, expectedDbPath);
});
} else {
throw new Error("neither staged replacement nor source backup exists");
}
Expand Down Expand Up @@ -612,7 +622,7 @@ export async function runLegacyTursoMigration(): Promise<void> {
mkdirSync(CONFIG.storagePath, { recursive: true });
}

recoverInterruptedReembedSwaps();
await recoverInterruptedReembedSwaps();

// Recovery may put a libSQL shard back at its active path. Convert DiskANN
// indexes before any legacy verification opens it with the Turso engine,
Expand Down
11 changes: 6 additions & 5 deletions src/services/turso/sqlite-handle-release.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ const GC_PASSES = 3;
const FILE_LOCK_RETRY_DELAYS_MS = [25, 50, 100, 200, 400, 800, 1600, 3200];
export const RETRYABLE_FILE_LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"]);

const SQLITE_SIDE_SUFFIXES = ["-wal", "-shm"] as const;
const SQLITE_SIDE_SUFFIXES = ["-wal", "-shm", "-tshm"] as const;

/**
* Rename a SQLite/Turso database file together with its WAL/SHM sidecars.
* Rename a SQLite/Turso database file together with its WAL/SHM/TSHM sidecars.
* Leaving a `.tmp-wal` behind after renaming only the main file drops commits.
* `-tshm` is Turso multiprocess WAL shared memory (multiprocess_wal).
*/
export function renameSqliteDatabase(fromPath: string, toPath: string): void {
renameSync(fromPath, toPath);
Expand All @@ -32,7 +33,7 @@ export function renameSqliteDatabase(fromPath: string, toPath: string): void {
}

/**
* Copy a SQLite/Turso database file together with its WAL/SHM sidecars.
* Copy a SQLite/Turso database file together with its WAL/SHM/TSHM sidecars.
*/
export function copySqliteDatabase(fromPath: string, toPath: string): void {
copyFileSync(fromPath, toPath);
Expand All @@ -49,10 +50,10 @@ export function copySqliteDatabase(fromPath: string, toPath: string): void {
}

/**
* Remove a SQLite/Turso database file together with WAL/SHM sidecars.
* Remove a SQLite/Turso database file together with WAL/SHM/TSHM sidecars.
*/
export function removeSqliteDatabase(dbPath: string): void {
for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) {
for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`, `${dbPath}-tshm`]) {
if (existsSync(path)) unlinkSync(path);
}
}
Expand Down
3 changes: 2 additions & 1 deletion tests/turso-connection-manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ describe("turso connection manager", () => {
expect(opens).toHaveLength(1);
expect(opens[0]?.path).toBe(dbPath);
expect(opens[0]?.opts).toMatchObject({
experimental: ["encryption"],
experimental:
process.platform === "win32" ? ["encryption"] : ["encryption", "multiprocess_wal"],
});
expect(opens[0]?.opts).not.toHaveProperty("encryption");
} finally {
Expand Down
Loading
Loading