diff --git a/packages/opencode/src/effect/app-runtime.ts b/packages/opencode/src/effect/app-runtime.ts index d68e00a32..5c306676c 100644 --- a/packages/opencode/src/effect/app-runtime.ts +++ b/packages/opencode/src/effect/app-runtime.ts @@ -8,7 +8,7 @@ import { Auth } from "@/auth" import { Account } from "@/account/account" import { Config } from "@/config" import { Git } from "@/git" -import { Ripgrep } from "@/file/ripgrep" +import { Search } from "@/file/search" import { File } from "@/file" import { FileWatcher } from "@/file/watcher" import { Storage } from "@/storage" @@ -57,7 +57,7 @@ export const AppLayer = Layer.mergeAll( Account.defaultLayer, Config.defaultLayer, Git.defaultLayer, - Ripgrep.defaultLayer, + Search.defaultLayer, File.defaultLayer, FileWatcher.defaultLayer, Storage.defaultLayer, diff --git a/packages/opencode/src/file/search.ts b/packages/opencode/src/file/search.ts new file mode 100644 index 000000000..df8f54fde --- /dev/null +++ b/packages/opencode/src/file/search.ts @@ -0,0 +1,481 @@ +import path from "path" +import { AppFileSystem } from "@opencode-ai/shared/filesystem" +import { Glob } from "@opencode-ai/shared/util/glob" +import { Context, Deferred, Effect, Layer, Option } from "effect" +import * as Stream from "effect/Stream" +import z from "zod" +import * as InstanceState from "@/effect/instance-state" +import { makeRuntime } from "@/effect/run-service" +import { Global } from "@/global" +import { Log } from "@/util" +import { Fff } from "#fff" +import { Ripgrep } from "./ripgrep" + +const log = Log.create({ service: "file.search" }) +const root = path.join(Global.Path.cache, "fff") + +export const Match = z.object({ + path: z.object({ + text: z.string(), + }), + lines: z.object({ + text: z.string(), + }), + line_number: z.number(), + absolute_offset: z.number(), + submatches: z.array( + z.object({ + match: z.object({ + text: z.string(), + }), + start: z.number(), + end: z.number(), + }), + ), +}) + +export type Item = z.infer + +export interface Result { + readonly items: Item[] + readonly partial: boolean + readonly engine: "fff" | "ripgrep" + readonly regexFallbackError?: string +} + +export interface FileInput { + readonly cwd: string + readonly query: string + readonly limit?: number + readonly current?: string +} + +export interface GlobInput { + readonly cwd: string + readonly pattern: string + readonly limit?: number + readonly signal?: AbortSignal +} + +interface Query { + readonly dir: string + readonly text: string + readonly files: string[] +} + +interface State { + readonly pick: Map + readonly wait: Map> + readonly recent: Query[] +} + +export interface Interface { + readonly files: Ripgrep.Interface["files"] + readonly tree: Ripgrep.Interface["tree"] + readonly search: (input: Ripgrep.SearchInput) => Effect.Effect + readonly file: (input: FileInput) => Effect.Effect + readonly glob: (input: GlobInput) => Effect.Effect<{ files: string[]; truncated: boolean }, unknown> + readonly open: (input: { cwd?: string; file: string }) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/Search") {} + +function key(dir: string) { + return Buffer.from(dir).toString("base64url") +} + +function normalize(text: string) { + return text.replaceAll("\\", "/") +} + +function blocked(rel: string) { + return normalize(rel).split("/").includes(".git") +} + +function basename(file: string) { + return normalize(file).split("/").at(-1) ?? file +} + +function allow(glob: string[] | undefined, rel: string, file: string) { + if (!glob?.length) return true + const include = glob.filter((item) => !item.startsWith("!")) + const exclude = glob.filter((item) => item.startsWith("!")).map((item) => item.slice(1)) + if (include.length > 0 && !include.some((item) => Glob.match(item, rel) || Glob.match(item, file))) return false + if (exclude.some((item) => Glob.match(item, rel) || Glob.match(item, file))) return false + return true +} + +function include(pattern: string) { + const value = pattern.trim().replaceAll("\\", "/") + if (!value) return "*" + const flat = value.replaceAll("**/", "").replaceAll("/**", "/") + const idx = flat.lastIndexOf("/") + if (idx < 0) return flat + const dir = flat.slice(0, idx + 1) + const glob = flat.slice(idx + 1) + if (!glob) return dir + return `${dir} ${glob}` +} + +function remember(state: State, dir: string, text: string, files: string[]) { + if (!files.length) return + const next = Array.from(new Set(files.map(AppFileSystem.resolve))).slice(0, 64) + if (!next.length) return + const idx = state.recent.findIndex((item) => item.dir === dir && item.text === text) + if (idx >= 0) state.recent.splice(idx, 1) + state.recent.unshift({ dir, text, files: next }) + if (state.recent.length > 32) state.recent.length = 32 +} + +function matchesTokens(query: string, file: string) { + const tokens = query.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean) + if (!tokens.length) return true + const normalized = normalize(file).toLowerCase() + return tokens.every((token) => normalized.includes(token)) +} + +function item(hit: Fff.Hit): Item { + return { + path: { text: normalize(hit.relativePath) }, + lines: { text: hit.lineContent }, + line_number: hit.lineNumber, + absolute_offset: hit.byteOffset, + submatches: hit.matchRanges + .map(([start, end]) => { + const text = hit.lineContent.slice(start, end) + if (!text) return undefined + return { + match: { text }, + start, + end, + } + }) + .filter((row): row is Item["submatches"][number] => Boolean(row)), + } +} + +export const layer: Layer.Layer = Layer.effect( + Service, + Effect.gen(function* () { + const fs = yield* AppFileSystem.Service + const rg = yield* Ripgrep.Service + const state = yield* InstanceState.make( + Effect.fn("Search.state")(() => + Effect.gen(function* () { + const next = { + pick: new Map(), + wait: new Map>(), + recent: [] as Query[], + } + yield* Effect.addFinalizer(() => + Effect.sync(() => { + for (const pick of next.pick.values()) pick.destroy() + }), + ) + return next + }), + ), + ) + + const rip = Effect.fn("Search.rip")(function* (input: Ripgrep.SearchInput) { + const out = yield* rg.search(input) + return { + items: out.items, + partial: out.partial, + engine: "ripgrep" as const, + } + }) + + const picker = Effect.fn("Search.picker")(function* (cwd: string) { + if (!Fff.available()) return undefined + + const dir = AppFileSystem.resolve(cwd) + const current = yield* InstanceState.get(state) + const existing = current.pick.get(dir) + if (existing) return existing + + const pending = current.wait.get(dir) + if (pending) return yield* Deferred.await(pending) + + const gate = yield* Deferred.make() + current.wait.set(dir, gate) + try { + yield* fs.ensureDir(root) + const id = key(dir) + const made = yield* Effect.sync(() => + Fff.create({ + basePath: dir, + frecencyDbPath: path.join(root, `${id}.frecency.mdb`), + historyDbPath: path.join(root, `${id}.history.mdb`), + aiMode: true, + }), + ) + if (!made.ok) { + log.warn("fff init failed", { dir, error: made.error }) + const err = new Error(made.error) + yield* Deferred.fail(gate, err) + return yield* Effect.fail(err) + } + + const pick = made.value + const ready = yield* Effect.sync(() => pick.waitForScan(5_000)) + if (!ready.ok) { + pick.destroy() + log.warn("fff scan failed", { dir, error: ready.error }) + const err = new Error(ready.error) + yield* Deferred.fail(gate, err) + return yield* Effect.fail(err) + } + if (!ready.value) { + pick.destroy() + const err = new Error("fff scan timed out") + log.warn("fff scan timed out", { dir }) + yield* Deferred.fail(gate, err) + return yield* Effect.fail(err) + } + + const git = yield* Effect.sync(() => pick.refreshGitStatus()) + if (!git.ok) log.warn("fff git refresh failed", { dir, error: git.error }) + + current.pick.set(dir, pick) + yield* Deferred.succeed(gate, pick) + return pick + } finally { + if (current.wait.get(dir) === gate) current.wait.delete(dir) + } + }) + + const files: Interface["files"] = (input) => rg.files(input) + const tree: Interface["tree"] = (input) => rg.tree(input) + + const file: Interface["file"] = Effect.fn("Search.file")(function* (input) { + const query = input.query.trim() + if (!query) return [] + + const pick = yield* picker(input.cwd).pipe(Effect.catch(() => Effect.succeed(undefined))) + if (!pick) return undefined + + const dir = AppFileSystem.resolve(input.cwd) + const out = yield* Effect.sync(() => + pick.fileSearch(query, { + currentFile: input.current + ? path.isAbsolute(input.current) + ? input.current + : path.join(dir, input.current) + : undefined, + pageIndex: 0, + pageSize: Math.max(input.limit ?? 100, 100), + }), + ) + if (!out.ok) { + log.warn("fff file search failed", { dir, query, error: out.error }) + return undefined + } + + const rows: string[] = Array.from( + new Set( + out.value.items.flatMap((item, idx): string[] => { + const score = out.value.scores[idx] + if (!score || score.total <= 0) return [] + return [normalize(item.relativePath)] + }), + ), + ) + const output = query.length >= 4 ? rows.filter((row) => matchesTokens(query, row)) : rows + const current = yield* InstanceState.get(state) + remember( + current, + dir, + query, + output.map((row) => path.join(dir, row)), + ) + return output.slice(0, input.limit ?? 100) + }) + + const search: Interface["search"] = Effect.fn("Search.search")(function* (input) { + input.signal?.throwIfAborted() + if (input.file?.length) return yield* rip(input) + + const pick = yield* picker(input.cwd).pipe(Effect.catch(() => Effect.succeed(undefined))) + if (!pick) return yield* rip(input) + + const dir = AppFileSystem.resolve(input.cwd) + const limit = input.limit ?? 100 + const rows: Item[] = [] + const seen = new Set() + let cursor: Fff.Cursor = null + let regexFallbackError: string | undefined + + while (rows.length < limit) { + input.signal?.throwIfAborted() + const out = yield* Effect.sync(() => + pick.grep(input.pattern, { + mode: "regex", + cursor, + maxMatchesPerFile: limit, + timeBudgetMs: 1_500, + }), + ) + if (!out.ok) { + log.warn("fff grep failed", { dir, pattern: input.pattern, error: out.error }) + return yield* rip(input) + } + + regexFallbackError = regexFallbackError ?? out.value.regexFallbackError + for (const hit of out.value.items) { + const rel = normalize(hit.relativePath) + if (!allow(input.glob, rel, normalize(hit.fileName))) continue + const id = `${rel}:${hit.lineNumber}:${hit.byteOffset}` + if (seen.has(id)) continue + seen.add(id) + rows.push(item(hit)) + if (rows.length >= limit) break + } + + if (!out.value.nextCursor) break + cursor = out.value.nextCursor + } + + if (!rows.length && input.glob?.length) return yield* rip(input) + + const current = yield* InstanceState.get(state) + remember( + current, + dir, + input.pattern, + Array.from(new Set(rows.map((row) => path.join(dir, row.path.text)))), + ) + + return { + items: rows, + partial: false, + engine: "fff" as const, + regexFallbackError, + } + }) + + const glob: Interface["glob"] = Effect.fn("Search.glob")(function* (input) { + input.signal?.throwIfAborted() + + const dir = AppFileSystem.resolve(input.cwd) + const limit = input.limit ?? 100 + const pick = yield* picker(dir).pipe(Effect.catch(() => Effect.succeed(undefined))) + if (pick) { + const out = yield* Effect.sync(() => + pick.fileSearch(include(input.pattern), { + currentFile: path.join(dir, ".opencode"), + pageIndex: 0, + pageSize: Math.max(limit * 4, 200), + }), + ) + if (out.ok) { + const rows: string[] = Array.from( + new Map( + out.value.items + .filter((item) => !blocked(item.relativePath)) + .filter( + (item) => + Glob.match(input.pattern, item.relativePath) || Glob.match(input.pattern, basename(item.relativePath)), + ) + .map((item) => [normalize(item.relativePath), item.modified] as const), + ).entries(), + ) + .sort((a, b) => b[1] - a[1]) + .map(([file]) => file) + + if (rows.length > 0) { + const current = yield* InstanceState.get(state) + remember( + current, + dir, + input.pattern, + rows.map((row) => path.join(dir, row)), + ) + return { + files: rows.slice(0, limit).map((row) => path.join(dir, row)), + truncated: rows.length > limit, + } + } + } else { + log.warn("fff glob failed", { dir, pattern: input.pattern, error: out.error }) + } + } + + const rows = yield* rg.files({ cwd: dir, glob: [input.pattern], signal: input.signal }).pipe( + Stream.take(limit + 1), + Stream.runCollect, + Effect.map((chunk) => [...chunk]), + ) + const truncated = rows.length > limit + if (truncated) rows.length = limit + + const output = yield* Effect.forEach( + rows, + Effect.fnUntraced(function* (file) { + const full = path.join(dir, file) + const info = yield* fs.stat(full).pipe(Effect.catch(() => Effect.succeed(undefined))) + const time = + info?.mtime.pipe( + Option.map((item) => item.getTime()), + Option.getOrElse(() => 0), + ) ?? 0 + return { file: full, time } + }), + { concurrency: 16 }, + ) + output.sort((a, b) => b.time - a.time) + return { + files: output.map((item) => item.file), + truncated, + } + }) + + const open: Interface["open"] = Effect.fn("Search.open")(function* (input) { + const current = yield* InstanceState.get(state) + const file = input.cwd + ? AppFileSystem.resolve(path.isAbsolute(input.file) ? input.file : path.join(input.cwd, input.file)) + : AppFileSystem.resolve(input.file) + const idx = current.recent.findIndex((item) => item.files.includes(file)) + if (idx < 0) return + + const row = current.recent[idx] + current.recent.splice(idx, 1) + const pick = current.pick.get(row.dir) + if (!pick) return + + const out = yield* Effect.sync(() => pick.trackQuery(row.text, file)) + if (!out.ok) log.warn("fff track query failed", { dir: row.dir, query: row.text, file, error: out.error }) + }) + + return Service.of({ files, tree, search, file, glob, open }) + }), +) + +export const defaultLayer: Layer.Layer = layer.pipe( + Layer.provide(Ripgrep.defaultLayer), + Layer.provide(AppFileSystem.defaultLayer), +) + +const { runPromise } = makeRuntime(Service, defaultLayer) + +export function tree(input: Ripgrep.TreeInput) { + return runPromise((svc) => svc.tree(input)) +} + +export function search(input: Ripgrep.SearchInput) { + return runPromise((svc) => svc.search(input)) +} + +export function file(input: FileInput) { + return runPromise((svc) => svc.file(input)) +} + +export function glob(input: GlobInput) { + return runPromise((svc) => svc.glob(input)) +} + +export function open(input: { cwd?: string; file: string }) { + return runPromise((svc) => svc.open(input)) +} + +export * as Search from "./search"