diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts index a8fcdfd73..3a681030c 100644 --- a/packages/core/src/database/database.ts +++ b/packages/core/src/database/database.ts @@ -3,21 +3,28 @@ export * as Database from "./database" import { EffectDrizzleSqlite } from "@opencode-ai/effect-drizzle-sqlite" import { layer as sqliteLayer } from "#sqlite" import { Context, Effect, Layer } from "effect" +import { Sqlite } from "./sqlite" import { Global } from "../global" import { Flag } from "../flag/flag" import { isAbsolute, join } from "path" import { DatabaseMigration } from "./migration" import { InstallationChannel } from "../installation/version" -import { makeRuntime } from "../effect/runtime" const makeDatabase = EffectDrizzleSqlite.makeWithDefaults() type DatabaseShape = Effect.Success +export type Info = { + db: DatabaseShape + native: unknown + drizzle: Sqlite.DrizzleClient +} -export class Service extends Context.Service()("@opencode/v2/storage/Database") {} +export class Service extends Context.Service()("@opencode/v2/storage/Database") {} -const layer = Layer.effect( +export const layer = Layer.effect( Service, Effect.gen(function* () { + const native = yield* Sqlite.Native + const drizzle = yield* Sqlite.Drizzle const db = yield* makeDatabase yield* db.run("PRAGMA journal_mode = WAL") @@ -29,12 +36,12 @@ const layer = Layer.effect( yield* Effect.log("Applying database migrations") yield* DatabaseMigration.apply(db) - return db + return { db, native, drizzle } }).pipe(Effect.orDie), ) export function layerFromPath(filename: string) { - return layer.pipe(Layer.provide(sqliteLayer({ filename })), Layer.orDie) + return layer.pipe(Layer.provide(sqliteLayer({ filename }))) } export const memoryLayer = layerFromPath(":memory:") @@ -58,7 +65,3 @@ export const defaultLayer = Layer.unwrap( return layerFromPath(path()) }), ).pipe(Layer.provide(Global.defaultLayer)) - -const { runSync } = makeRuntime(Service, defaultLayer) - -export const init = () => runSync(() => Effect.void) diff --git a/packages/core/src/database/migration.ts b/packages/core/src/database/migration.ts index db2ddd10f..42aebf02d 100644 --- a/packages/core/src/database/migration.ts +++ b/packages/core/src/database/migration.ts @@ -1,80 +1,51 @@ export * as DatabaseMigration from "./migration" -import { sql, type SQLWrapper } from "drizzle-orm" +import { sql } from "drizzle-orm" import { Effect } from "effect" import type { EffectDrizzleSqlite } from "@opencode-ai/effect-drizzle-sqlite" import { migrations } from "./migration.gen" -type EffectDatabase = EffectDrizzleSqlite.EffectSQLiteDatabase -type Query = string | SQLWrapper -type MigrationEffect = Effect.Effect -export type Transaction = { - run: (query: Query) => MigrationEffect -} -type SyncDatabase = { - run: (query: Query) => unknown - all: (query: Query) => A[] - get: (query: Query) => A | undefined -} -type SyncTransaction = { - run: (query: Query) => unknown -} -type Database = EffectDatabase | SyncDatabase -type Target = { - run: (query: Query) => MigrationEffect - all: (query: Query) => MigrationEffect - get: (query: Query) => MigrationEffect - transaction: (body: (tx: Transaction) => MigrationEffect) => MigrationEffect -} +type Database = EffectDrizzleSqlite.EffectSQLiteDatabase +type Transaction = Parameters[0]>[0] export type Migration = { id: string - up: (tx: Transaction) => MigrationEffect + up: (tx: Transaction) => Effect.Effect } -export function apply(db: EffectDatabase): MigrationEffect -export function apply(db: SyncDatabase): MigrationEffect export function apply(db: Database) { - return applyOnlyImpl(db, migrations) + return applyOnly(db, migrations) } -export function applyOnly(db: EffectDatabase, input: Migration[]): MigrationEffect -export function applyOnly(db: SyncDatabase, input: Migration[]): MigrationEffect export function applyOnly(db: Database, input: Migration[]) { - return applyOnlyImpl(db, input) -} - -function applyOnlyImpl(db: Database, input: Migration[]) { return Effect.gen(function* () { - const target = normalize(db) - - yield* target.run( + yield* db.run( sql`CREATE TABLE IF NOT EXISTS ${sql.identifier("migration")} (id TEXT PRIMARY KEY, time_completed INTEGER NOT NULL)`, ) let completed = new Set( - (yield* target.all<{ id: string }>(sql`SELECT id FROM ${sql.identifier("migration")}`)).map((row) => row.id), + (yield* db.all<{ id: string }>(sql`SELECT id FROM ${sql.identifier("migration")}`)).map((row) => row.id), ) if (completed.size === 0) { // Existing installs used Drizzle's migration journal. Seed the new // journal once so TypeScript migrations don't replay old SQL. if ( - yield* target.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ${"__drizzle_migrations"}`) + yield* db.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ${"__drizzle_migrations"}`) ) { - yield* target.run(sql` + yield* db.run(sql` INSERT OR IGNORE INTO ${sql.identifier("migration")} (id, time_completed) SELECT name, ${Date.now()} FROM ${sql.identifier("__drizzle_migrations")} WHERE name IS NOT NULL `) completed = new Set( - (yield* target.all<{ id: string }>(sql`SELECT id FROM ${sql.identifier("migration")}`)).map((row) => row.id), + (yield* db.all<{ id: string }>(sql`SELECT id FROM ${sql.identifier("migration")}`)).map((row) => row.id), ) } } for (const migration of input) { if (completed.has(migration.id)) continue - yield* target.transaction((tx) => + yield* db.transaction((tx) => Effect.gen(function* () { yield* migration.up(tx) yield* tx.run( @@ -85,49 +56,3 @@ function applyOnlyImpl(db: Database, input: Migration[]) { } }) } - -function normalize(db: Database): Target { - if (isEffectDatabase(db)) return normalizeEffect(db) - return normalizeSync(db) -} - -function normalizeEffect(db: EffectDatabase): Target { - return { - run: (query) => db.run(query).pipe(Effect.as(undefined)), - all: (query) => db.all(query), - get: (query) => db.get(query), - transaction: (body) => db.transaction((tx) => body(normalizeEffectTransaction(tx))), - } -} - -function normalizeSync(db: SyncDatabase): Target { - const tx = normalizeSyncTransaction(db) - return { - run: tx.run, - all: (query) => Effect.try({ try: () => db.all(query), catch: (err) => err }), - get: (query) => Effect.try({ try: () => db.get(query), catch: (err) => err }), - transaction: (body) => - Effect.gen(function* () { - yield* tx.run("BEGIN") - const result = yield* body(tx).pipe(Effect.catch((err) => tx.run("ROLLBACK").pipe(Effect.flatMap(() => Effect.fail(err))))) - yield* tx.run("COMMIT") - return result - }), - } -} - -function normalizeEffectTransaction(tx: { run: (query: Query) => MigrationEffect }): Transaction { - return { - run: (query) => tx.run(query), - } -} - -function normalizeSyncTransaction(tx: SyncTransaction): Transaction { - return { - run: (query) => Effect.try({ try: () => tx.run(query), catch: (err) => err }), - } -} - -function isEffectDatabase(db: Database): db is EffectDatabase { - return "raw" in db -} diff --git a/packages/core/src/database/sqlite.bun.ts b/packages/core/src/database/sqlite.bun.ts index 735fcd49d..28801e789 100644 --- a/packages/core/src/database/sqlite.bun.ts +++ b/packages/core/src/database/sqlite.bun.ts @@ -1,3 +1,175 @@ -import { SqliteClient } from "@effect/sql-sqlite-bun" +import { Database } from "bun:sqlite" +import { drizzle } from "drizzle-orm/bun-sqlite" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Fiber from "effect/Fiber" +import { identity } from "effect/Function" +import * as Layer from "effect/Layer" +import * as Scope from "effect/Scope" +import * as Semaphore from "effect/Semaphore" +import * as Stream from "effect/Stream" +import * as Reactivity from "effect/unstable/reactivity/Reactivity" +import * as Client from "effect/unstable/sql/SqlClient" +import type { Connection } from "effect/unstable/sql/SqlConnection" +import { classifySqliteError, SqlError } from "effect/unstable/sql/SqlError" +import * as Statement from "effect/unstable/sql/Statement" +import { Sqlite } from "./sqlite" -export const layer = SqliteClient.layer +const ATTR_DB_SYSTEM_NAME = "db.system.name" + +const TypeId = "~@opencode-ai/core/database/SqliteBun" as const +type TypeId = typeof TypeId + +interface SqliteClient extends Client.SqlClient { + readonly [TypeId]: TypeId + readonly config: Config + readonly export: Effect.Effect + readonly loadExtension: (path: string) => Effect.Effect + readonly updateValues: never +} + +interface Config { + readonly filename: string + readonly readonly?: boolean + readonly create?: boolean + readonly readwrite?: boolean + readonly disableWAL?: boolean + readonly spanAttributes?: Record + readonly transformResultNames?: (str: string) => string + readonly transformQueryNames?: (str: string) => string +} + +interface SqliteConnection extends Connection { + readonly export: Effect.Effect + readonly loadExtension: (path: string) => Effect.Effect +} + +const make = (options: Config) => + Effect.gen(function* () { + const native = (yield* Sqlite.Native) as Database + + const compiler = Statement.makeCompilerSqlite(options.transformQueryNames) + const transformRows = options.transformResultNames ? Statement.defaultTransforms(options.transformResultNames).array : undefined + + const run = (query: string, params: ReadonlyArray = []) => + Effect.withFiber>, SqlError>((fiber) => { + const statement = native.query(query) + // @ts-ignore bun-types missing safeIntegers method, fixed in https://github.com/oven-sh/bun/pull/26627 + statement.safeIntegers(Context.get(fiber.context, Client.SafeIntegers)) + try { + return Effect.succeed((statement.all(...(params as any)) ?? []) as Array>) + } catch (cause) { + return Effect.fail( + new SqlError({ + reason: classifySqliteError(cause, { message: "Failed to execute statement", operation: "execute" }), + }), + ) + } + }) + + const runValues = (query: string, params: ReadonlyArray = []) => + Effect.withFiber, SqlError>((fiber) => { + const statement = native.query(query) + // @ts-ignore bun-types missing safeIntegers method, fixed in https://github.com/oven-sh/bun/pull/26627 + statement.safeIntegers(Context.get(fiber.context, Client.SafeIntegers)) + try { + return Effect.succeed((statement.values(...(params as any)) ?? []) as Array) + } catch (cause) { + return Effect.fail( + new SqlError({ + reason: classifySqliteError(cause, { message: "Failed to execute statement", operation: "execute" }), + }), + ) + } + }) + + const connection = identity({ + execute(query, params, transformRows) { + return transformRows ? Effect.map(run(query, params), transformRows) : run(query, params) + }, + executeRaw(query, params) { + return run(query, params) + }, + executeValues(query, params) { + return runValues(query, params) + }, + executeUnprepared(query, params, transformRows) { + return this.execute(query, params, transformRows) + }, + executeStream() { + return Stream.die("executeStream not implemented") + }, + export: Effect.try({ + try: () => native.serialize(), + catch: (cause) => + new SqlError({ reason: classifySqliteError(cause, { message: "Failed to export database", operation: "export" }) }), + }), + loadExtension: (path) => + Effect.try({ + try: () => native.loadExtension(path), + catch: (cause) => + new SqlError({ + reason: classifySqliteError(cause, { message: "Failed to load extension", operation: "loadExtension" }), + }), + }), + }) + + const semaphore = yield* Semaphore.make(1) + const acquirer = semaphore.withPermits(1)(Effect.succeed(connection)) + const transactionAcquirer = Effect.uninterruptibleMask((restore) => { + const fiber = Fiber.getCurrent()! + const scope = Context.getUnsafe(fiber.context, Scope.Scope) + return Effect.as(Effect.tap(restore(semaphore.take(1)), () => Scope.addFinalizer(scope, semaphore.release(1))), connection) + }) + + const client = Object.assign( + (yield* Client.make({ + acquirer, + compiler, + transactionAcquirer, + spanAttributes: [ + ...(options.spanAttributes ? Object.entries(options.spanAttributes) : []), + [ATTR_DB_SYSTEM_NAME, "sqlite"], + ], + transformRows, + })) as SqliteClient, + { + [TypeId]: TypeId, + config: options, + export: Effect.flatMap(acquirer, (_) => _.export), + loadExtension: (path: string) => Effect.flatMap(acquirer, (_) => _.loadExtension(path)), + }, + ) + + return client + }) + +const nativeLayer = (config: Config) => + Layer.effect( + Sqlite.Native, + Effect.gen(function* () { + const native = new Database(config.filename, { + readonly: config.readonly, + readwrite: config.readwrite ?? true, + create: config.create ?? true, + }) + yield* Effect.addFinalizer(() => Effect.sync(() => native.close())) + if (config.disableWAL !== true) native.run("PRAGMA journal_mode = WAL;") + return native + }), + ) + +const sqliteLayer = (config: Config) => Layer.effect(Client.SqlClient, make(config)) + +const drizzleLayer = Layer.effect( + Sqlite.Drizzle, + Effect.gen(function* () { + return drizzle({ client: (yield* Sqlite.Native) as Database }) + }), +) + +export const layer = (config: Config) => + Layer.merge( + nativeLayer(config), + Layer.merge(sqliteLayer(config), drizzleLayer).pipe(Layer.provide(nativeLayer(config))), + ).pipe(Layer.provide(Reactivity.layer)) diff --git a/packages/core/src/database/sqlite.node.ts b/packages/core/src/database/sqlite.node.ts index 150ed6c94..4f1057ea0 100644 --- a/packages/core/src/database/sqlite.node.ts +++ b/packages/core/src/database/sqlite.node.ts @@ -1,3 +1,170 @@ -import { NodeSqliteClient } from "@opencode-ai/effect-sqlite-node" +import { DatabaseSync, type SQLInputValue } from "node:sqlite" +import { drizzle } from "drizzle-orm/node-sqlite" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Fiber from "effect/Fiber" +import { identity } from "effect/Function" +import * as Layer from "effect/Layer" +import * as Scope from "effect/Scope" +import * as Semaphore from "effect/Semaphore" +import * as Stream from "effect/Stream" +import * as Reactivity from "effect/unstable/reactivity/Reactivity" +import * as Client from "effect/unstable/sql/SqlClient" +import type { Connection } from "effect/unstable/sql/SqlConnection" +import { classifySqliteError, SqlError } from "effect/unstable/sql/SqlError" +import * as Statement from "effect/unstable/sql/Statement" +import { Sqlite } from "./sqlite" -export const layer = NodeSqliteClient.layer +const ATTR_DB_SYSTEM_NAME = "db.system.name" + +const TypeId = "~@opencode-ai/core/database/SqliteNode" as const +type TypeId = typeof TypeId + +interface SqliteClient extends Client.SqlClient { + readonly [TypeId]: TypeId + readonly config: Config + readonly loadExtension: (path: string) => Effect.Effect + readonly updateValues: never +} + +interface Config { + readonly filename: string + readonly readonly?: boolean + readonly create?: boolean + readonly readwrite?: boolean + readonly disableWAL?: boolean + readonly timeout?: number + readonly allowExtension?: boolean + readonly spanAttributes?: Record + readonly transformResultNames?: (str: string) => string + readonly transformQueryNames?: (str: string) => string +} + +interface SqliteConnection extends Connection { + readonly loadExtension: (path: string) => Effect.Effect +} + +const make = (options: Config) => + Effect.gen(function* () { + const native = (yield* Sqlite.Native) as DatabaseSync + + const compiler = Statement.makeCompilerSqlite(options.transformQueryNames) + const transformRows = options.transformResultNames ? Statement.defaultTransforms(options.transformResultNames).array : undefined + + const run = (query: string, params: ReadonlyArray = []) => + Effect.withFiber>, SqlError>((fiber) => { + const statement = native.prepare(query) + statement.setReadBigInts(Context.get(fiber.context, Client.SafeIntegers)) + try { + return Effect.succeed(statement.all(...(params as SQLInputValue[])) as Array>) + } catch (cause) { + return Effect.fail( + new SqlError({ + reason: classifySqliteError(cause, { message: "Failed to execute statement", operation: "execute" }), + }), + ) + } + }) + + const runValues = (query: string, params: ReadonlyArray = []) => + Effect.withFiber>, SqlError>((fiber) => { + const statement = native.prepare(query) + statement.setReadBigInts(Context.get(fiber.context, Client.SafeIntegers)) + statement.setReturnArrays(true) + try { + return Effect.succeed(statement.all(...(params as SQLInputValue[])) as unknown as ReadonlyArray>) + } catch (cause) { + return Effect.fail( + new SqlError({ + reason: classifySqliteError(cause, { message: "Failed to execute statement", operation: "execute" }), + }), + ) + } + }) + + const connection = identity({ + execute(query, params, transformRows) { + return transformRows ? Effect.map(run(query, params), transformRows) : run(query, params) + }, + executeRaw(query, params) { + return run(query, params) + }, + executeValues(query, params) { + return runValues(query, params) + }, + executeUnprepared(query, params, transformRows) { + return this.execute(query, params, transformRows) + }, + executeStream() { + return Stream.die("executeStream not implemented") + }, + loadExtension: (path) => + Effect.try({ + try: () => native.loadExtension(path), + catch: (cause) => + new SqlError({ + reason: classifySqliteError(cause, { message: "Failed to load extension", operation: "loadExtension" }), + }), + }), + }) + + const semaphore = yield* Semaphore.make(1) + const acquirer = semaphore.withPermits(1)(Effect.succeed(connection)) + const transactionAcquirer = Effect.uninterruptibleMask((restore) => { + const fiber = Fiber.getCurrent()! + const scope = Context.getUnsafe(fiber.context, Scope.Scope) + return Effect.as(Effect.tap(restore(semaphore.take(1)), () => Scope.addFinalizer(scope, semaphore.release(1))), connection) + }) + + const client = Object.assign( + (yield* Client.make({ + acquirer, + compiler, + transactionAcquirer, + spanAttributes: [ + ...(options.spanAttributes ? Object.entries(options.spanAttributes) : []), + [ATTR_DB_SYSTEM_NAME, "sqlite"], + ], + transformRows, + })) as SqliteClient, + { + [TypeId]: TypeId, + config: options, + loadExtension: (path: string) => Effect.flatMap(acquirer, (_) => _.loadExtension(path)), + }, + ) + + return client + }) + +const nativeLayer = (config: Config) => + Layer.effect( + Sqlite.Native, + Effect.gen(function* () { + const native = new DatabaseSync(config.filename, { + readOnly: config.readonly, + timeout: config.timeout, + allowExtension: config.allowExtension, + enableForeignKeyConstraints: true, + open: true, + }) + yield* Effect.addFinalizer(() => Effect.sync(() => native.close())) + if (config.disableWAL !== true && config.readonly !== true) native.exec("PRAGMA journal_mode = WAL;") + return native + }), + ) + +const sqliteLayer = (config: Config) => Layer.effect(Client.SqlClient, make(config)) + +const drizzleLayer = Layer.effect( + Sqlite.Drizzle, + Effect.gen(function* () { + return drizzle({ client: (yield* Sqlite.Native) as DatabaseSync }) as unknown as Sqlite.DrizzleClient + }), +) + +export const layer = (config: Config) => + Layer.merge( + nativeLayer(config), + Layer.merge(sqliteLayer(config), drizzleLayer).pipe(Layer.provide(nativeLayer(config))), + ).pipe(Layer.provide(Reactivity.layer)) diff --git a/packages/core/src/database/sqlite.ts b/packages/core/src/database/sqlite.ts new file mode 100644 index 000000000..cba003b4c --- /dev/null +++ b/packages/core/src/database/sqlite.ts @@ -0,0 +1,14 @@ +export * as Sqlite from "./sqlite" + +import { Context } from "effect" +import type { drizzle } from "drizzle-orm/bun-sqlite" + +export type DrizzleClient = ReturnType + +export interface Info { + native: Native + drizzle: DrizzleClient +} + +export class Native extends Context.Service()("@opencode-ai/core/database/SqliteNative") {} +export class Drizzle extends Context.Service()("@opencode-ai/core/database/SqliteDrizzle") {} diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index 74f3a80c7..edf2be744 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -171,7 +171,7 @@ function fromRow(row: typeof SessionTable.$inferSelect): SessionSchema.Info { export const layer = Layer.effect( Service, Effect.gen(function* () { - const db = yield* Database.Service + const db = (yield* Database.Service).db const decodeMessage = Schema.decodeUnknownEffect(SessionMessage.Message) const decode = (row: typeof SessionMessageTable.$inferSelect) => diff --git a/packages/opencode/src/storage/db.ts b/packages/opencode/src/storage/db.ts index a6e7c0079..66b1d44c9 100644 --- a/packages/opencode/src/storage/db.ts +++ b/packages/opencode/src/storage/db.ts @@ -1,25 +1,27 @@ import { type SQLiteTransaction } from "drizzle-orm/sqlite-core" +import type { TablesRelationalConfig } from "drizzle-orm/relations" export * from "drizzle-orm" import { LocalContext } from "@/util/local-context" import * as Log from "@opencode-ai/core/util/log" import { NamedError } from "@opencode-ai/core/util/error" import { EffectBridge } from "@/effect/bridge" -import { init } from "#db" import { Effect, Schema } from "effect" import { Database } from "@opencode-ai/core/database/database" -import { DatabaseMigration } from "@opencode-ai/core/database/migration" +import { makeRuntime } from "@opencode-ai/core/effect/runtime" export const NotFoundError = NamedError.create("NotFoundError", { message: Schema.String, }) const log = Log.create({ service: "db" }) +const runtime = makeRuntime(Database.Service, Database.defaultLayer) +const database = await runtime.runPromise((db) => Effect.succeed(db)) export const getPath = () => Database.path() -export type Transaction = SQLiteTransaction<"sync", void> +export type Transaction = SQLiteTransaction<"sync", void, Record, TablesRelationalConfig> -type Client = ReturnType +type Client = Database.Info["drizzle"] let client: Client | undefined let loaded = false @@ -31,7 +33,7 @@ export const Client = Object.assign( const dbPath = getPath() log.info("opening database", { path: dbPath }) - const db = init(dbPath) + const db = database.drizzle db.run("PRAGMA journal_mode = WAL") db.run("PRAGMA synchronous = NORMAL") @@ -39,7 +41,6 @@ export const Client = Object.assign( db.run("PRAGMA cache_size = -64000") db.run("PRAGMA foreign_keys = ON") db.run("PRAGMA wal_checkpoint(PASSIVE)") - Effect.runSync(DatabaseMigration.apply(db)) client = db loaded = true @@ -56,7 +57,6 @@ export const Client = Object.assign( export function close() { if (!Client.loaded()) return - client?.$client.close() Client.reset() }