mattpocock-skills-opencode/scripts/sync-upstream.ts

// Updates the pristine upstream/ snapshot from mattpocock/skills and records the revision.
//
//   node scripts/sync-upstream.ts [path-to-existing-clone]
//
// Without a path it shallow-clones https://github.com/mattpocock/skills into a temp dir.
// Exits 0 with "no changes" when upstream HEAD equals the recorded commit and the snapshot exists,
// so the nightly workflow can stop early. On change it replaces upstream/skills wholesale, writes
// upstream/UPSTREAM.json, updates .opencode/state/upstream.json and adds needs-review entries for
// new skills in .opencode/state/classification.json, then prints the added/removed/changed skills.
import { execFileSync } from "node:child_process"
import { createHash } from "node:crypto"
import {
  cpSync,
  existsSync,
  mkdtempSync,
  readFileSync,
  readdirSync,
  rmSync,
  statSync,
  writeFileSync,
} from "node:fs"
import { tmpdir } from "node:os"
import path from "node:path"
import { fileURLToPath } from "node:url"

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
export const UPSTREAM_URL = "https://github.com/mattpocock/skills"
const STATE_PATH = path.join(root, ".opencode/state/upstream.json")
const CLASSIFICATION_PATH = path.join(root, ".opencode/state/classification.json")
const SNAPSHOT_DIR = path.join(root, "upstream")

export type UpstreamState = {
  repo: string
  commit: string
  commit_date: string
  upstream_version: string
  synced_at: string | null
  note?: string
}

/** Names of skill dirs (those containing SKILL.md) under a skills/ root, as "<bucket>/<name>". */
export function listSkillDirs(skillsRoot: string): string[] {
  const out: string[] = []
  if (!existsSync(skillsRoot)) return out
  for (const bucket of readdirSync(skillsRoot).sort()) {
    const bucketPath = path.join(skillsRoot, bucket)
    if (!statSync(bucketPath).isDirectory()) continue
    for (const name of readdirSync(bucketPath).sort()) {
      if (existsSync(path.join(bucketPath, name, "SKILL.md"))) out.push(`${bucket}/${name}`)
    }
  }
  return out
}

/** Deterministic content hash of a directory (sorted relative paths + file bytes). */
export function hashDir(dir: string): string {
  const hash = createHash("sha256")
  const walk = (d: string, prefix: string) => {
    for (const entry of readdirSync(d).sort()) {
      const full = path.join(d, entry)
      const rel = prefix ? `${prefix}/${entry}` : entry
      if (statSync(full).isDirectory()) walk(full, rel)
      else hash.update(rel).update(readFileSync(full))
    }
  }
  walk(dir, "")
  return hash.digest("hex")
}

export type SnapshotDiff = { added: string[]; removed: string[]; changed: string[] }

export function diffSnapshots(before: string, after: string): SnapshotDiff {
  const oldDirs = new Map(
    listSkillDirs(before).map((s) => [s, hashDir(path.join(before, ...s.split("/")))]),
  )
  const newDirs = new Map(
    listSkillDirs(after).map((s) => [s, hashDir(path.join(after, ...s.split("/")))]),
  )
  const diff: SnapshotDiff = { added: [], removed: [], changed: [] }
  for (const [name, hash] of newDirs) {
    if (!oldDirs.has(name)) diff.added.push(name)
    else if (oldDirs.get(name) !== hash) diff.changed.push(name)
  }
  for (const name of oldDirs.keys()) if (!newDirs.has(name)) diff.removed.push(name)
  return diff
}

/** Appends needs-review classification entries for skill dirs missing from classification.json. */
export function recordNewSkills(classificationPath: string, added: string[]): string[] {
  if (added.length === 0) return []
  const doc = JSON.parse(readFileSync(classificationPath, "utf8"))
  const known = new Set(doc.skills.map((s: { name: string }) => s.name))
  const fresh: string[] = []
  for (const dir of added) {
    const [bucket, name] = dir.split("/")
    if (known.has(name)) continue
    doc.skills.push({
      name,
      bucket,
      classification: "skill",
      user_invoked: false,
      artifacts: ["skill:verbatim"],
      depends_on: [],
      status: "needs-review",
      rationale: "New upstream skill; default classification, awaiting human review.",
    })
    fresh.push(name)
  }
  if (fresh.length) writeFileSync(classificationPath, JSON.stringify(doc, null, 2) + "\n")
  return fresh
}

function git(cloneDir: string, ...args: string[]): string {
  return execFileSync("git", ["-C", cloneDir, ...args], { encoding: "utf8" }).trim()
}

async function main() {
  const existing = process.argv[2]
  let cloneDir = existing
  let temp: string | undefined
  if (!cloneDir) {
    temp = mkdtempSync(path.join(tmpdir(), "mp-skills-"))
    cloneDir = path.join(temp, "skills")
    console.log(`cloning ${UPSTREAM_URL} ...`)
    execFileSync("git", ["clone", "--quiet", "--depth", "1", UPSTREAM_URL, cloneDir])
  }
  try {
    const commit = git(cloneDir, "rev-parse", "HEAD")
    const commitDate = git(cloneDir, "log", "-1", "--format=%cI")
    const upstreamPkg = JSON.parse(readFileSync(path.join(cloneDir, "package.json"), "utf8"))

    const recorded: UpstreamState | undefined = existsSync(STATE_PATH)
      ? JSON.parse(readFileSync(STATE_PATH, "utf8"))
      : undefined
    if (
      recorded?.commit === commit &&
      existsSync(path.join(SNAPSHOT_DIR, "skills")) &&
      recorded.synced_at
    ) {
      console.log(`no changes: upstream already at ${commit.slice(0, 9)}`)
      return
    }

    const oldSkills = path.join(SNAPSHOT_DIR, "skills")
    const diff = existsSync(oldSkills)
      ? diffSnapshots(oldSkills, path.join(cloneDir, "skills"))
      : { added: listSkillDirs(path.join(cloneDir, "skills")), removed: [], changed: [] }

    rmSync(oldSkills, { recursive: true, force: true })
    cpSync(path.join(cloneDir, "skills"), oldSkills, { recursive: true })
    const meta = {
      repo: UPSTREAM_URL,
      commit,
      commit_date: commitDate,
      upstream_version: upstreamPkg.version,
    }
    writeFileSync(path.join(SNAPSHOT_DIR, "UPSTREAM.json"), JSON.stringify(meta, null, 2) + "\n")
    const state: UpstreamState & { previous_commit?: string | null; last_diff?: SnapshotDiff } = {
      ...meta,
      synced_at: new Date().toISOString(),
      previous_commit: recorded?.commit ?? null,
      last_diff: diff,
    }
    writeFileSync(STATE_PATH, JSON.stringify(state, null, 2) + "\n")

    const fresh = recordNewSkills(CLASSIFICATION_PATH, diff.added)
    console.log(`synced upstream to ${commit.slice(0, 9)} (${commitDate}), v${upstreamPkg.version}`)
    console.log(
      `skills: +${diff.added.length} -${diff.removed.length} ~${diff.changed.length}` +
        (diff.added.length ? `\n  added:   ${diff.added.join(", ")}` : "") +
        (diff.removed.length ? `\n  removed: ${diff.removed.join(", ")}` : "") +
        (diff.changed.length ? `\n  changed: ${diff.changed.join(", ")}` : ""),
    )
    if (fresh.length) {
      console.log(`classification: added needs-review entries for ${fresh.join(", ")}`)
      console.log("NOTE: needs-review entries fail the drift test until reviewed.")
    }
  } finally {
    if (temp) rmSync(temp, { recursive: true, force: true })
  }
}

if (process.argv[1] === fileURLToPath(import.meta.url)) {
  main().catch((err) => {
    console.error(err.message)
    process.exit(1)
  })
}