diff --git a/packages/core/src/config/reference.ts b/packages/core/src/config/reference.ts index dc9042e6f..fbd6c840d 100644 --- a/packages/core/src/config/reference.ts +++ b/packages/core/src/config/reference.ts @@ -15,3 +15,34 @@ export const Entry = Schema.Union([Schema.String, Git, Local]) export type Entry = typeof Entry.Type export const Info = Schema.Record(Schema.String, Entry) +export type Info = typeof Info.Type + +export type NormalizedEntry = + | { readonly kind: "local"; readonly path: string } + | { readonly kind: "git"; readonly repository: string; readonly branch?: string } + | { readonly kind: "invalid"; readonly message: string } + +export type NormalizedInfo = Record + +export function validateAlias(name: string) { + if (name.length === 0) return "Reference alias must not be empty" + if (/[\/\s`,]/.test(name)) return "Reference alias must not contain /, whitespace, comma, or backtick" +} + +export function normalizeEntry(entry: Entry): NormalizedEntry { + if (typeof entry === "string") { + if (entry.startsWith(".") || entry.startsWith("/") || entry.startsWith("~")) return { kind: "local", path: entry } + return { kind: "git", repository: entry } + } + if ("path" in entry) return { kind: "local", path: entry.path } + return { kind: "git", repository: entry.repository, branch: entry.branch } +} + +export function normalize(info: Info): NormalizedInfo { + return Object.fromEntries( + Object.entries(info).map(([name, entry]) => { + const message = validateAlias(name) + return [name, message ? { kind: "invalid" as const, message } : normalizeEntry(entry)] + }), + ) +} diff --git a/packages/core/src/flag/flag.ts b/packages/core/src/flag/flag.ts index c9269b9c2..ee9228e37 100644 --- a/packages/core/src/flag/flag.ts +++ b/packages/core/src/flag/flag.ts @@ -5,11 +5,10 @@ function truthy(key: string) { return value === "true" || value === "1" } -const OPENCODE_EXPERIMENTAL = truthy("OPENCODE_EXPERIMENTAL") const copy = process.env["OPENCODE_EXPERIMENTAL_DISABLE_COPY_ON_SELECT"] function enabledByExperimental(key: string) { - return process.env[key] === undefined ? OPENCODE_EXPERIMENTAL : truthy(key) + return process.env[key] === undefined ? truthy("OPENCODE_EXPERIMENTAL") : truthy(key) } export const Flag = { @@ -54,6 +53,9 @@ export const Flag = { get OPENCODE_DISABLE_PROJECT_CONFIG() { return truthy("OPENCODE_DISABLE_PROJECT_CONFIG") }, + get OPENCODE_EXPERIMENTAL_REFERENCES() { + return enabledByExperimental("OPENCODE_EXPERIMENTAL_REFERENCES") + }, get OPENCODE_TUI_CONFIG() { return process.env["OPENCODE_TUI_CONFIG"] }, diff --git a/packages/core/src/location-layer.ts b/packages/core/src/location-layer.ts index c4d9ea544..e95c65aa9 100644 --- a/packages/core/src/location-layer.ts +++ b/packages/core/src/location-layer.ts @@ -18,6 +18,8 @@ import { PermissionV2 } from "./permission" import { PermissionSaved } from "./permission/saved" import { SessionV2 } from "./session" import { LocationFileSystem } from "./location-filesystem" +import { ProjectReference } from "./project-reference" +import { RepositoryCache } from "./repository-cache" export class LocationServiceMap extends LayerMap.Service()("@opencode/example/LocationServiceMap", { lookup: (ref: Location.Ref) => { @@ -26,6 +28,7 @@ export class LocationServiceMap extends LayerMap.Service()(" location, Policy.locationLayer, Config.locationLayer, + ProjectReference.locationLayer, PluginV2.locationLayer, Catalog.locationLayer, AgentV2.locationLayer, @@ -46,5 +49,6 @@ export class LocationServiceMap extends LayerMap.Service()(" Database.defaultLayer, SessionV2.defaultLayer, PermissionSaved.defaultLayer, + RepositoryCache.defaultLayer, ], }) {} diff --git a/packages/core/src/project-reference.ts b/packages/core/src/project-reference.ts new file mode 100644 index 000000000..84b659d52 --- /dev/null +++ b/packages/core/src/project-reference.ts @@ -0,0 +1,208 @@ +export * as ProjectReference from "./project-reference" + +import path from "path" +import { Context, Effect, Layer } from "effect" +import { Config } from "./config" +import { ConfigReference } from "./config/reference" +import { AppFileSystem } from "./filesystem" +import { Flag } from "./flag/flag" +import { Global } from "./global" +import { Location } from "./location" +import { Repository } from "./repository" +import { RepositoryCache } from "./repository-cache" + +export type Resolved = + | { readonly name: string; readonly kind: "local"; readonly path: string } + | { + readonly name: string + readonly kind: "git" + readonly repository: string + readonly reference: Repository.RemoteReference + readonly path: string + readonly branch?: string + } + | { readonly name: string; readonly kind: "invalid"; readonly repository?: string; readonly message: string } + +type Valid = Exclude + +export type Mention = + | { readonly name: string; readonly kind: "reference"; readonly reference: Valid; readonly target?: string; readonly path: string } + | { readonly name: string; readonly kind: "invalid"; readonly target?: string; readonly message: string } + | { readonly name: string; readonly kind: "missing"; readonly target: string; readonly path: string; readonly message: string } + +export interface Interface { + readonly list: () => Effect.Effect + readonly get: (name: string) => Effect.Effect + readonly resolveMention: (value: string) => Effect.Effect + readonly ensurePath: (target?: string) => Effect.Effect + readonly containsManagedPath: (target?: string) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/ProjectReference") {} + +type Materializer = { + readonly name: string + readonly repository: string + readonly path: string + readonly run: Effect.Effect +} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + if (!Flag.OPENCODE_EXPERIMENTAL_REFERENCES) return Service.of(inert) + + const config = yield* Config.Service + const fs = yield* AppFileSystem.Service + const global = yield* Global.Service + const location = yield* Location.Service + const cache = yield* RepositoryCache.Service + const references = resolveAll({ + references: ConfigReference.normalize( + Object.assign({}, ...(yield* config.get()).map((document) => document.info.references ?? {})), + ), + directory: location.project.directory, + home: global.home, + repos: global.repos, + }) + const materializers = yield* Effect.forEach( + uniqueGitReferences(references), + Effect.fnUntraced(function* (reference) { + return { + name: reference.name, + repository: reference.repository, + path: reference.path, + run: yield* Effect.cached( + cache.ensure({ reference: reference.reference, branch: reference.branch, refresh: true }).pipe(Effect.asVoid), + ), + } + }), + ) + + yield* Effect.forEach( + materializers, + (materializer) => + materializer.run.pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to materialize project reference").pipe( + Effect.annotateLogs({ name: materializer.name, repository: materializer.repository, cause }), + ), + ), + ), + { concurrency: 4, discard: true }, + ).pipe( + Effect.forkScoped, + ) + + const ensurePath = Effect.fn("ProjectReference.ensurePath")(function* (target?: string) { + const normalized = normalizePath(target) + if (!normalized) return yield* Effect.forEach(materializers, (materializer) => materializer.run, { discard: true }) + yield* materializers.find((materializer) => contains(materializer.path, normalized))?.run ?? Effect.void + }) + + return Service.of({ + list: Effect.fn("ProjectReference.list")(function* () { + return references + }), + get: Effect.fn("ProjectReference.get")(function* (name: string) { + return references.find((reference) => reference.name === name) + }), + ensurePath, + containsManagedPath: Effect.fn("ProjectReference.containsManagedPath")(function* (target?: string) { + const normalized = normalizePath(target) + return normalized ? references.some((reference) => reference.kind === "git" && contains(reference.path, normalized)) : false + }), + resolveMention: Effect.fn("ProjectReference.resolveMention")(function* (value: string) { + const [name, ...rest] = value.split("/") + const target = rest.length ? rest.join("/") : undefined + const reference = references.find((reference) => reference.name === name) + if (!reference) return + if (reference.kind === "invalid") return { name, kind: "invalid", target, message: reference.message } + if (reference.kind === "git") yield* ensurePath(reference.path) + if (!target) return { name, kind: "reference", reference, path: reference.path } + + const resolved = path.resolve(reference.path, target) + if (!AppFileSystem.contains(reference.path, resolved)) return { name, kind: "invalid", target, message: "Reference target escapes its root" } + if (!(yield* fs.existsSafe(resolved))) return { name, kind: "missing", target, path: resolved, message: "Reference target does not exist" } + return { name, kind: "reference", reference, target, path: resolved } + }), + }) + }), +) + +export const locationLayer = layer.pipe(Layer.provideMerge(Config.locationLayer)) + +const inert: Interface = { + list: () => Effect.succeed([]), + get: () => Effect.succeed(undefined), + resolveMention: () => Effect.succeed(undefined), + ensurePath: () => Effect.void, + containsManagedPath: () => Effect.succeed(false), +} + +export function resolveAll(input: { references: ConfigReference.NormalizedInfo; directory: string; home: string; repos: string }) { + const seen = new Map() + return Object.entries(input.references).map(([name, reference]): Resolved => { + const resolved = resolve({ name, reference, directory: input.directory, home: input.home, repos: input.repos }) + if (resolved.kind !== "git") return resolved + const existing = seen.get(resolved.path) + if (!existing) { + seen.set(resolved.path, { name, branch: resolved.branch }) + return resolved + } + if (existing.branch === resolved.branch) return resolved + return { + name, + kind: "invalid", + repository: resolved.repository, + message: `Reference conflicts with @${existing.name}: both use ${resolved.path}, but @${existing.name} requests ${existing.branch ?? "default branch"} and @${name} requests ${resolved.branch ?? "default branch"}`, + } + }) +} + +export function resolve(input: { name: string; reference: ConfigReference.NormalizedEntry; directory: string; home: string; repos: string }): Resolved { + if (input.reference.kind === "invalid") return { name: input.name, kind: "invalid", message: input.reference.message } + if (input.reference.kind === "local") { + return { name: input.name, kind: "local", path: localPath(input.directory, input.home, input.reference.path) } + } + const reference = Repository.parse(input.reference.repository) + if (!reference || !Repository.isRemote(reference)) { + return { + name: input.name, + kind: "invalid", + repository: input.reference.repository, + message: "Repository must be a git URL, host/path reference, or GitHub owner/repo shorthand", + } + } + return { + name: input.name, + kind: "git", + repository: input.reference.repository, + reference, + path: Repository.cachePath(input.repos, reference), + branch: input.reference.branch, + } +} + +function localPath(directory: string, home: string, value: string) { + if (value.startsWith("~/")) return path.join(home, value.slice(2)) + return path.isAbsolute(value) ? value : path.resolve(directory, value) +} + +function uniqueGitReferences(references: Resolved[]) { + const seen = new Set() + return references.filter((reference): reference is Extract => { + if (reference.kind !== "git" || seen.has(reference.path)) return false + seen.add(reference.path) + return true + }) +} + +function normalizePath(target?: string) { + if (!target) return + return process.platform === "win32" ? AppFileSystem.normalizePath(target) : target +} + +function contains(parent: string, child: string) { + return AppFileSystem.contains(normalizePath(parent) ?? parent, normalizePath(child) ?? child) +} diff --git a/packages/core/test/location-layer.test.ts b/packages/core/test/location-layer.test.ts index 19ab93121..4555be253 100644 --- a/packages/core/test/location-layer.test.ts +++ b/packages/core/test/location-layer.test.ts @@ -16,6 +16,7 @@ import { Global } from "../src/global" import { ModelsDev } from "../src/models-dev" import { Npm } from "../src/npm" import { Project } from "../src/project" +import { ProjectReference } from "../src/project-reference" const it = testEffect( LocationServiceMap.layer.pipe( @@ -53,6 +54,7 @@ describe("LocationServiceMap", () => { const update = (directory: string) => Effect.gen(function* () { yield* PluginBoot.Service.use((boot) => boot.wait()) + yield* ProjectReference.Service const catalog = yield* Catalog.Service const transform = yield* catalog.transform() yield* transform((editor) => editor.provider.update(ProviderV2.ID.make("test"), () => {})) diff --git a/packages/core/test/project-reference.test.ts b/packages/core/test/project-reference.test.ts new file mode 100644 index 000000000..081e4117b --- /dev/null +++ b/packages/core/test/project-reference.test.ts @@ -0,0 +1,267 @@ +import { describe, expect } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Deferred, Effect, Layer, Schema } from "effect" +import { Config } from "@opencode-ai/core/config" +import { ConfigReference } from "@opencode-ai/core/config/reference" +import { AppFileSystem } from "@opencode-ai/core/filesystem" +import { Flag } from "@opencode-ai/core/flag/flag" +import { Global } from "@opencode-ai/core/global" +import { Location } from "@opencode-ai/core/location" +import { ProjectReference } from "@opencode-ai/core/project-reference" +import { Repository } from "@opencode-ai/core/repository" +import { RepositoryCache } from "@opencode-ai/core/repository-cache" +import { AbsolutePath } from "@opencode-ai/core/schema" +import { location } from "./fixture/location" +import { tmpdir } from "./fixture/tmpdir" +import { it } from "./lib/effect" + +describe("ProjectReference", () => { + it.live("uses the broad experimental flag unless references are explicitly configured", () => + withEnv({ OPENCODE_EXPERIMENTAL: "true", OPENCODE_EXPERIMENTAL_REFERENCES: undefined }, + Effect.sync(() => { + expect(Flag.OPENCODE_EXPERIMENTAL_REFERENCES).toBe(true) + }), + ).pipe( + Effect.flatMap(() => + withEnv({ OPENCODE_EXPERIMENTAL: "true", OPENCODE_EXPERIMENTAL_REFERENCES: "false" }, + Effect.sync(() => { + expect(Flag.OPENCODE_EXPERIMENTAL_REFERENCES).toBe(false) + }), + ), + ), + ), + ) + + it.live("normalizes aliases and resolves relative local paths from the project root", () => + withTmp((tmp) => + Effect.gen(function* () { + const project = path.join(tmp.path, "project") + const nested = path.join(project, "packages", "app") + yield* Effect.promise(() => fs.mkdir(nested, { recursive: true })) + + const references = ProjectReference.resolveAll({ + references: ConfigReference.normalize({ + docs: { path: "./docs" }, + home: "~/notes", + sdk: { repository: "owner/repo", branch: "main" }, + shorthand: "owner/other", + invalid: "not-a-repo", + "bad/name": "owner/repo", + }), + directory: project, + home: path.join(tmp.path, "home"), + repos: path.join(tmp.path, "repos"), + }) + + expect(references).toMatchObject([ + { name: "docs", kind: "local", path: path.join(project, "docs") }, + { name: "home", kind: "local", path: path.join(tmp.path, "home", "notes") }, + { name: "sdk", kind: "git", branch: "main" }, + { name: "shorthand", kind: "git" }, + { name: "invalid", kind: "invalid", repository: "not-a-repo" }, + { name: "bad/name", kind: "invalid" }, + ]) + }), + ), + ) + + it.live("marks same-cache references with different branches invalid", () => + Effect.sync(() => { + const references = ProjectReference.resolveAll({ + references: ConfigReference.normalize({ + main: { repository: "owner/repo", branch: "main" }, + dev: { repository: "github.com/owner/repo", branch: "dev" }, + alsoMain: { repository: "https://github.com/owner/repo", branch: "main" }, + }), + directory: "/project", + home: "/home", + repos: "/repos", + }) + + expect(references.map((reference) => reference.kind)).toEqual(["git", "invalid", "git"]) + expect(references[1]?.kind === "invalid" ? references[1].message : "").toContain("conflicts with @main") + }), + ) + + it.live("merges config aliases and exposes mention and managed-path operations", () => + withoutReferences(withTmp((tmp) => { + const calls: RepositoryCache.EnsureInput[] = [] + const project = path.join(tmp.path, "project") + const nested = path.join(project, "packages", "app") + const docs = path.join(project, "docs") + const repos = path.join(tmp.path, "repos") + return Effect.gen(function* () { + yield* Effect.promise(async () => { + await fs.mkdir(nested, { recursive: true }) + await fs.mkdir(docs) + await fs.writeFile(path.join(docs, "README.md"), "docs") + }) + + yield* withReferences( + Effect.gen(function* () { + const references = yield* ProjectReference.Service + const git = path.join(repos, "github.com", "owner", "repo") + + expect(yield* references.list()).toMatchObject([ + { name: "docs", kind: "local", path: docs }, + { name: "sdk", kind: "git", path: git }, + ]) + expect(yield* references.resolveMention("docs/README.md")).toMatchObject({ + name: "docs", + kind: "reference", + target: "README.md", + path: path.join(docs, "README.md"), + }) + expect(yield* references.resolveMention("docs/missing.md")).toMatchObject({ name: "docs", kind: "missing" }) + expect(yield* references.resolveMention("docs/../outside.md")).toMatchObject({ name: "docs", kind: "invalid" }) + expect(yield* references.resolveMention("unknown")).toBeUndefined() + expect(yield* references.resolveMention("sdk")).toMatchObject({ name: "sdk", kind: "reference", path: git }) + expect(yield* references.containsManagedPath(path.join(git, "README.md"))).toBe(true) + expect(yield* references.containsManagedPath(path.join(docs, "README.md"))).toBe(false) + yield* references.ensurePath() + expect(calls).toHaveLength(1) + }).pipe( + Effect.provide( + testLayer({ + directory: nested, + project, + repos, + documents: [ + document({ docs: { path: "./old-docs" }, sdk: "owner/old" }), + document({ docs: { path: "./docs" }, sdk: { repository: "owner/repo", branch: "main" } }), + ], + ensure: (input) => Effect.sync(() => result(repos, calls, input)), + }), + ), + ), + ) + }) + })), + ) + + it.live("is inert while the runtime flag is disabled", () => + withoutReferences(withTmp((tmp) => { + const calls: RepositoryCache.EnsureInput[] = [] + return Effect.gen(function* () { + const references = yield* ProjectReference.Service + expect(yield* references.list()).toEqual([]) + expect(yield* references.get("sdk")).toBeUndefined() + expect(yield* references.resolveMention("sdk")).toBeUndefined() + expect(yield* references.containsManagedPath(path.join(tmp.path, "repos", "github.com", "owner", "repo"))).toBe(false) + yield* references.ensurePath() + expect(calls).toEqual([]) + }).pipe( + Effect.provide( + testLayer({ + directory: tmp.path, + project: tmp.path, + repos: path.join(tmp.path, "repos"), + documents: [document({ sdk: "owner/repo" })], + ensure: (input) => Effect.sync(() => result(path.join(tmp.path, "repos"), calls, input)), + }), + ), + ) + })), + ) + + it.live("starts Git materialization in the background without blocking the location layer", () => + withTmp((tmp) => + Effect.gen(function* () { + const started = yield* Deferred.make() + yield* withReferences( + Effect.gen(function* () { + expect(yield* (yield* ProjectReference.Service).list()).toHaveLength(1) + yield* Deferred.await(started).pipe( + Effect.timeoutOrElse({ duration: "1 second", orElse: () => Effect.die(new Error("refresh did not start")) }), + ) + }).pipe( + Effect.provide( + testLayer({ + directory: tmp.path, + project: tmp.path, + repos: path.join(tmp.path, "repos"), + documents: [document({ sdk: "owner/repo" })], + ensure: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)), + }), + ), + ), + ) + }), + ), + ) +}) + +function document(references: ConfigReference.Info) { + return new Config.Loaded({ source: { type: "memory" }, info: Schema.decodeUnknownSync(Config.Info)({ references }) }) +} + +function result(repos: string, calls: RepositoryCache.EnsureInput[], input: RepositoryCache.EnsureInput): RepositoryCache.Result { + calls.push(input) + return { + repository: input.reference.label, + host: input.reference.host, + remote: input.reference.remote, + localPath: Repository.cachePath(repos, input.reference), + status: "cached", + branch: input.branch, + } +} + +function testLayer(input: { + directory: string + project: string + repos: string + documents: Config.Loaded[] + ensure: RepositoryCache.Interface["ensure"] +}) { + return ProjectReference.layer.pipe( + Layer.provide( + Layer.mergeAll( + AppFileSystem.defaultLayer, + Global.layerWith({ home: path.join(input.directory, "home"), repos: input.repos }), + Layer.succeed( + Location.Service, + Location.Service.of( + location({ directory: AbsolutePath.make(input.directory) }, { projectDirectory: AbsolutePath.make(input.project) }), + ), + ), + Layer.succeed(Config.Service, Config.Service.of({ directories: () => Effect.succeed([]), get: () => Effect.succeed(input.documents) })), + Layer.succeed(RepositoryCache.Service, RepositoryCache.Service.of({ ensure: input.ensure })), + ), + ), + ) +} + +function withTmp(body: (tmp: Awaited>) => Effect.Effect) { + return Effect.acquireUseRelease(Effect.promise(() => tmpdir()), body, (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]())) +} + +function withReferences(body: Effect.Effect) { + return withEnv({ OPENCODE_EXPERIMENTAL_REFERENCES: "true" }, body) +} + +function withoutReferences(body: Effect.Effect) { + return withEnv({ OPENCODE_EXPERIMENTAL: undefined, OPENCODE_EXPERIMENTAL_REFERENCES: undefined }, body) +} + +function withEnv(env: Record, body: Effect.Effect) { + return Effect.acquireUseRelease( + Effect.sync(() => { + const previous = Object.fromEntries(Object.keys(env).map((key) => [key, process.env[key]])) + for (const [key, value] of Object.entries(env)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + return previous + }), + () => body, + (previous) => + Effect.sync(() => { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }), + ) +}