| author | Divy Srivastava <me@littledivy.com> | 2026-08-19 13:47:03 +0530 |
|---|---|---|
| committer | Divy Srivastava <me@littledivy.com> | 2026-08-19 13:47:03 +0530 |
| commit | 4fffc3a9fd6d04293af3494faf44c85d523a4dc4 (patch) | |
| tree | 99e9f1005f8ee27391abd718aea3c8a2c6b3d295 | |
| parent | 0c364ef2f62ad36c5e72598973f2f44a17eb926c (diff) | |
| download | dgit-4fffc3a9fd.tar.gz zip | |
v0.0.2
Diffstat
| README.md | +23 | -7 |
| src/env.ts | +7 | -1 |
| src/git/multipart.ts (new) | +106 | -0 |
| src/git/packstore.ts | +279 | -105 |
| src/git/protocol.ts | +220 | -61 |
| src/git/store.ts | +11 | -2 |
| src/index.ts | +79 | -1 |
| src/repo.ts | +255 | -157 |
| wrangler.jsonc | +6 | -0 |
9 files changed, 986 insertions(+), 334 deletions(-)
diff --git a/README.md b/README.md @@ -23,8 +23,13 @@ A push streams into the repository's cell and is stored as the packfile the client sent; an index maps each object id to its pack, offset, and delta base, so the client's compression is preserved rather than -re-derived. A clone walks the closure of the requested refs and copies -the stored compressed bytes verbatim into the outgoing pack. Fetch +re-derived. When an R2 bucket is bound the pack bytes are written to R2 +and only the index stays in the cell's SQLite, so pack storage is no +longer capped by the per-cell database. A clone walks the closure of the +requested refs and copies the stored compressed bytes verbatim into the +outgoing pack; a full clone, once built, is cached in R2 and afterwards +streamed straight from the Worker, so repeat clones never load the cell. +Fetch negotiation excludes the closure of the client's haves, cut correctly at shallow boundaries, so an incremental fetch downloads only what is missing. Shallow clones (`--depth`, deepening, `--unshallow`), thin @@ -44,10 +49,16 @@ ```sh npm install +npx wrangler r2 bucket create dgit-pack-cache # optional: R2 pack offload + clone cache npx wrangler deploy npx wrangler secret put GIT_TOKEN # the push password ``` +The `PACK_CACHE` R2 binding in `wrangler.jsonc` is optional: with it, +pushed pack bytes live in R2 (off the cell's SQLite) and full clones are +served straight from R2 by the Worker without loading the cell. Remove +the binding and everything falls back to the SQLite-only path. + Then push anything: ```sh @@ -57,10 +68,12 @@ A Workers request is bounded at 128MB of memory and five minutes of CPU, so a very large history lands as a series of smaller pushes rather -than one; day-to-day pushes, clones, and fetches fit comfortably. A -full-history clone of a repository with millions of objects can exceed -the CPU bound — shallow and incremental fetches of the same repository -are fine. +than one; day-to-day pushes, clones, and fetches fit comfortably. +Building a full-history clone of a repository with millions of objects +can exceed the CPU bound the first time — once such a clone is cached in +R2 it streams from the Worker without rebuilding, and shallow and +incremental fetches of the same repository are fine regardless. The +largest repositories belong on celld. ## Self-host on celld @@ -91,7 +104,10 @@ Garbage collection also runs by itself, from a Durable Object alarm, after a forced update or a ref deletion. `GIT_TOKENS` holds additional -comma-separated tokens; `MAX_PUSH_MB` caps a single push. +comma-separated tokens; `MAX_PUSH_MB` caps a single push. Setting +`SHA1DC=1` screens every pushed object for a SHA-1 collision attack on +ingest; by default objects are hashed with native SHA-1 — the same +object ids, without the check. ## Contributions diff --git a/src/env.ts b/src/env.ts @@ -1,8 +1,11 @@ import type { Registry } from "./registry"; +import type { RepoCell } from "./repo"; export interface Env { - REPO: DurableObjectNamespace; + REPO: DurableObjectNamespace<RepoCell>; REGISTRY: DurableObjectNamespace<Registry>; + /** optional R2 bucket that offloads full-clone packs (absent on celld) */ + PACK_CACHE?: R2Bucket; /** shared secret required for git push (Basic auth password) */ GIT_TOKEN?: string; /** optional comma-separated extra tokens (multiple users) */ @@ -9,6 +12,9 @@ GIT_TOKENS?: string; /** max accepted push size in MB (default 512) */ MAX_PUSH_MB?: string; + /** "1" screens every pushed object with SHA-1DC collision detection; default + * hashes with native crypto.subtle (identical oid, far faster) */ + SHA1DC?: string; SITE_NAME?: string; SITE_DESC?: string; SITE_OWNER?: string; diff --git a/src/git/multipart.ts b/src/git/multipart.ts @@ -0,0 +1,106 @@ +import { concat } from "./util"; + +/** In-progress R2 multipart upload for one pack object (clone cache or raw store). */ +export interface MultipartPackUpload { + uploadPart(data: Uint8Array): Promise<void>; + complete(): Promise<void>; + abort(): Promise<void>; +} + +/** Uniform multipart part size. R2 requires every part but the last to be the + * same size and >= 5 MiB; 16 MiB parts cap a pack at 16 MiB * 10,000 = 160 GiB + * while holding at most one part in isolate memory at a time. */ +export const R2_PART_SIZE = 16 * 1024 * 1024; + +/** + * Buffers a streamed pack into fixed-size R2 multipart parts. Chunks are copied + * on push (callers hand out views into buffers that later reads evict). `drain` + * uploads every whole 16 MiB part and keeps the sub-part remainder, so isolate + * memory never holds more than ~one part; `finish` flushes the final (short) + * part and completes; any failure aborts so R2 is left with no partial object. + * Uniform part size (exactly R2_PART_SIZE, last excepted) is what satisfies R2's + * equal-size-parts rule. + */ +export class MultipartCapture { + private chunks: Uint8Array[] = []; + private len = 0; + private failed = false; + private completed = false; + constructor(private mp: MultipartPackUpload) {} + + /** True once a part upload or complete/abort has failed (no R2 object exists). */ + get aborted(): boolean { + return this.failed; + } + + push(chunk: Uint8Array): void { + if (this.failed) return; + this.chunks.push(chunk.slice()); + this.len += chunk.length; + } + + async drain(): Promise<void> { + while (!this.failed && this.len >= R2_PART_SIZE) { + const merged = concat(this.chunks); + try { + await this.mp.uploadPart(merged.subarray(0, R2_PART_SIZE)); + } catch { + await this.abort(); + return; + } + const rest = merged.slice(R2_PART_SIZE); + this.chunks = rest.length ? [rest] : []; + this.len = rest.length; + } + } + + async finish(): Promise<void> { + if (this.failed) return; + try { + if (this.len) await this.mp.uploadPart(concat(this.chunks)); + await this.mp.complete(); + this.completed = true; + } catch { + await this.abort(); + } + this.chunks = []; + this.len = 0; + } + + async abort(): Promise<void> { + if (this.failed || this.completed) return; + this.failed = true; + this.chunks = []; + this.len = 0; + try { + await this.mp.abort(); + } catch { + // best effort: an un-aborted multipart auto-expires + } + } +} + +/** Begin a raw R2 multipart upload under `key`. Returns null (caller falls back) + * if R2 refuses — celld deliberately makes createMultipartUpload throw. */ +export async function beginRawMultipart( + bucket: R2Bucket, + key: string +): Promise<MultipartPackUpload | null> { + try { + const mp = await bucket.createMultipartUpload(key); + const parts: R2UploadedPart[] = []; + return { + uploadPart: async (data) => { + parts.push(await mp.uploadPart(parts.length + 1, data)); + }, + complete: async () => { + await mp.complete(parts); + }, + abort: async () => { + await mp.abort(); + }, + }; + } catch { + return null; + } +} diff --git a/src/git/packstore.ts b/src/git/packstore.ts @@ -3,6 +3,7 @@ import { Sha1Dc, Sha1CollisionError } from "./sha1"; import { ObjType, NUM_TYPE, objectHeader } from "./objects"; import { applyDelta } from "./pack"; +import { MultipartCapture, beginRawMultipart } from "./multipart"; export const PACK_CHUNK = 1024 * 1024; /** real Workers isolates have a hard 128MB total; self-hosted celld nodes run multi-GB heaps */ @@ -21,11 +22,27 @@ const RAW_CHUNK_CACHE = TIGHT_MEMORY ? 4 : 8; /** * Hard cap on delta base-chain length when resolving an object. git's default - * pack depth is 50; 100 leaves generous headroom for legitimately deep packs - * while a crafted deeper (or cyclic) chain is rejected instead of overflowing - * the stack or looping forever. + * pack depth is 50; 50000 leaves enormous headroom for legitimately deep packs + * while a crafted unbounded chain is rejected instead of looping forever. The + * base walk is iterative (not recursive) and separately cycle-checked via a + * seen-set, so this bounds chain length rather than guarding stack depth. */ -const MAX_DELTA_DEPTH = 100; +const MAX_DELTA_DEPTH = 50000; +/** + * pako output-chunk size. The default (64 KiB) is large enough that V8 keeps + * every inflate's backing store in its array-buffer arena, so streaming a pack + * one Inflate per object commits arena proportional to the object count — a + * 53MB pack peaked near 250MB of buffers this way. 16 KiB chunks are collected + * promptly, holding inflate memory to a bounded working set regardless of pack + * size. + */ +const INFLATE_CHUNK = 16 * 1024; +const inflateAll = (data: Uint8Array): Uint8Array => { + const inf = new pako.Inflate({ chunkSize: INFLATE_CHUNK }); + inf.push(data, true); + if (inf.err) throw new Error(`inflate failed: ${inf.msg}`); + return inf.result as Uint8Array; +}; export interface ObjRec { type: ObjType; @@ -75,12 +92,15 @@ } } -// ingest is strictly sequential, so scratch hashers serve every object +// ingest is strictly sequential, so scratch hashers serve every object. The +// SHA-1DC pair is used only when collision detection is enabled; the plain +// streaming hasher covers the flag-off huge-object path. const scratchSha = new Sha1Dc(); const streamSha = new Sha1Dc(); +const streamShaPlain = new Sha1(); /** - * Finish an object hash, refusing anything carrying a SHA-1 collision-attack + * Finish a SHA-1DC hash, refusing anything carrying a SHA-1 collision-attack * block. This is git's post-SHAttered rule: the id is still plain SHA-1, but * an object built to collide never enters the database. */ @@ -91,64 +111,103 @@ } /** - * Every ingested object goes through the collision-detecting hash, so there - * is no fast path here: crypto.subtle is ~10x quicker on large blobs but - * cannot screen for near-collision blocks, and an attacker would only have to - * pad past the threshold to skip the check. + * Hash a fully-materialized object. With collision detection off (default) the + * id comes from native crypto.subtle — byte-identical to plain SHA-1 but ~24x + * faster than the JS hasher; with it on, SHA-1DC screens every block. The + * object is already in memory here, so the one-shot header+data buffer adds no + * unbounded allocation. */ -async function hashObjectAsync(type: ObjType, data: Uint8Array): Promise<string> { - return finishOid(scratchSha.reset().update(objectHeader(type, data.length)).update(data)); +async function hashObjectAsync(type: ObjType, data: Uint8Array, cd: boolean): Promise<string> { + if (cd) return finishOid(scratchSha.reset().update(objectHeader(type, data.length)).update(data)); + const header = objectHeader(type, data.length); + const buf = new Uint8Array(header.length + data.length); + buf.set(header); + buf.set(data, header.length); + return toHex(new Uint8Array(await crypto.subtle.digest("SHA-1", buf))); } -/** Buffered sequential reader over a pack stored as chunk rows. */ +/** + * Buffered sequential reader over a pack stored as SQLite chunk rows or an R2 + * object. Chunks are loaded through the owning PackStore's chunk cache (async + * because an R2-backed chunk is a range GET), so byte()/window() await; the SQL + * backend resolves without I/O. + */ class PackReader { pos = 0; private seq = -1; private chunk: Uint8Array = new Uint8Array(0); - constructor(private sql: SqlStorage, private packId: number, readonly size: number) {} + constructor(private load: (seq: number) => Promise<Uint8Array>, readonly size: number) {} - private ensure(): void { + private async ensure(): Promise<void> { const want = Math.floor(this.pos / PACK_CHUNK); if (want !== this.seq) { - const rows = this.sql - .exec<{ data: ArrayBuffer }>( - "SELECT data FROM pack_data WHERE pack_id = ? AND seq = ?", - this.packId, - want - ) - .toArray(); - if (!rows.length) throw new Error(`pack ${this.packId}: missing chunk ${want}`); - this.chunk = new Uint8Array(rows[0].data); + this.chunk = await this.load(want); this.seq = want; } } - byte(): number { - this.ensure(); + async byte(): Promise<number> { + await this.ensure(); return this.chunk[this.pos++ - this.seq * PACK_CHUNK]; } /** Remaining bytes of the chunk containing pos (never empty while pos < size). */ - window(): Uint8Array { - this.ensure(); + async window(): Promise<Uint8Array> { + await this.ensure(); return this.chunk.subarray(this.pos - this.seq * PACK_CHUNK); } } /** - * Pack-native object database: received packfiles are stored verbatim in - * chunk rows and indexed (oid -> pack/offset/base), preserving the client's - * delta compression. This is what lets Linux-sized repos fit and stream. + * Pack-native object database: received packfiles are stored verbatim — in R2 + * under `raw/<repo>/<packId>` when a bucket is bound (PACK_CACHE), else in + * SQLite chunk rows (the fallback / celld path) — and indexed (oid -> + * pack/offset/base), preserving the client's delta compression. Only the index + * ever lives in SQLite for an R2-backed pack, so pack data escapes the per-DO + * storage ceiling. This is what lets large repos fit and stream. */ export class PackStore { - /** LRU of decoded `pack_data` rows, keyed "packId:seq". */ + /** LRU of decoded pack chunks, keyed "packId:seq" (SQLite row or R2 range). */ private chunks = new Map<string, Uint8Array>(); + /** + * R2 raw-pack backend for this repo, or null when no bucket is bound (celld / + * Workers without the binding). When set, newly ingested packs store their + * bytes in R2 under `raw/<repo>/<packId>` and keep only the index in SQLite; + * pre-existing SQLite packs keep working (backend is recorded per pack). + */ + private r2: { bucket: R2Bucket; repo: string } | null = null; + /** Cached backend ('sql' | 'r2') per pack id, from pack_meta.store. */ + private backend = new Map<number, "sql" | "r2">(); constructor(private sql: SqlStorage, private extern: ExternalResolver) { this.init(); } + /** Bind (or clear) the R2 backend. Idempotent; the DO is per-repo so this is + * set on each request from the repo name. */ + setR2(bucket: R2Bucket | undefined, repo: string): void { + this.r2 = bucket ? { bucket, repo } : null; + } + + private rawKey(packId: number): string { + return `raw/${this.r2!.repo}/${packId}`; + } + + /** Storage backend of a pack. Cached; falls back to a pack_meta lookup. The + * cache is primed at ingest start (before pack_meta exists) so phase B/C read + * the right source. */ + private backendOf(packId: number): "sql" | "r2" { + const hit = this.backend.get(packId); + if (hit) return hit; + const rows = this.sql + .exec<{ store: string }>("SELECT store FROM pack_meta WHERE pack_id = ?", packId) + .toArray(); + const b = rows[0]?.store === "r2" ? "r2" : "sql"; + this.backend.set(packId, b); + return b; + } + private init(): void { this.sql.exec(` CREATE TABLE IF NOT EXISTS pack_meta ( @@ -155,7 +214,8 @@ pack_id INTEGER PRIMARY KEY, size INTEGER NOT NULL, count INTEGER NOT NULL, - created INTEGER NOT NULL + created INTEGER NOT NULL, + store TEXT NOT NULL DEFAULT 'sql' ); CREATE TABLE IF NOT EXISTS pack_data ( pack_id INTEGER NOT NULL, @@ -186,6 +246,12 @@ PRIMARY KEY (pack_id, offset) ); `); + // back-fill the store column on repos whose pack_meta predates the R2 backend: + // every such pack is SQLite-backed, which the 'sql' default records + const cols = this.sql.exec<{ name: string }>("PRAGMA table_info(pack_meta)").toArray(); + if (!cols.some((c) => c.name === "store")) { + this.sql.exec("ALTER TABLE pack_meta ADD COLUMN store TEXT NOT NULL DEFAULT 'sql'"); + } } wipe(): void { @@ -193,6 +259,7 @@ this.sql.exec(`DROP TABLE IF EXISTS ${t}`); } this.chunks.clear(); // pack ids restart from 1: cached rows would be stale + this.backend.clear(); } /** Drop all packs and start empty (small-repo gc migrates objects out first). */ @@ -201,6 +268,27 @@ this.init(); } + /** Pack ids of the currently R2-backed packs (for GC to delete their R2 bytes). */ + r2PackIds(): number[] { + return this.sql + .exec<{ pack_id: number }>("SELECT pack_id FROM pack_meta WHERE store = 'r2'") + .toArray() + .map((r) => r.pack_id); + } + + /** Delete the R2 raw objects for the given pack ids (no-op without a binding). */ + async deleteR2Packs(packIds: number[]): Promise<void> { + if (!this.r2 || !packIds.length) return; + const keys = packIds.map((id) => this.rawKey(id)); + for (let i = 0; i < keys.length; i += 100) { + try { + await this.r2.bucket.delete(keys.slice(i, i + 100)); + } catch { + // best effort: an undeleted raw object is unreferenced garbage + } + } + } + countObjects(): number { return this.sql.exec<{ n: number }>("SELECT COUNT(*) AS n FROM pack_objects").one().n; } @@ -265,12 +353,12 @@ * fetched one at a time: a wide range in a single SELECT can exceed the * runtime's result-set size cap (celld materializes blob rows as JSON). */ - readRaw(packId: number, off: number, len: number): Uint8Array { + async readRaw(packId: number, off: number, len: number): Promise<Uint8Array> { const first = Math.floor(off / PACK_CHUNK); const last = Math.floor((off + len - 1) / PACK_CHUNK); const out = new Uint8Array(len); for (let seq = first; seq <= last; seq++) { - const chunk = this.chunk(packId, seq); + const chunk = await this.chunk(packId, seq); const chunkStart = seq * PACK_CHUNK; const from = Math.max(off, chunkStart); const to = Math.min(off + len, chunkStart + chunk.length); @@ -280,12 +368,15 @@ } /** - * One decoded `pack_data` row, through the LRU. Rows are immutable once - * written — a pack chunk is inserted exactly once and only ever removed - * wholesale (wipe/reset, or the orphan sweep at the top of ingest, both of - * which clear the cache), so a hit can never be stale. + * One decoded pack chunk, through the LRU. For a SQLite-backed pack this is a + * `pack_data` row; for an R2-backed pack it is an aligned ~1 MiB range GET + * against `raw/<repo>/<packId>`, so sorted sequential reads (a clone) issue + * ~one GET per chunk instead of one per object. Chunks are immutable once + * written — a pack is inserted once and only ever removed wholesale (wipe/ + * reset, or the orphan sweep at the top of ingest, both of which clear the + * cache), so a hit can never be stale. */ - private chunk(packId: number, seq: number): Uint8Array { + private async chunk(packId: number, seq: number): Promise<Uint8Array> { const key = `${packId}:${seq}`; const hit = this.chunks.get(key); if (hit) { @@ -293,15 +384,25 @@ this.chunks.set(key, hit); return hit; } - const rows = this.sql - .exec<{ data: ArrayBuffer }>( - "SELECT data FROM pack_data WHERE pack_id = ? AND seq = ?", - packId, - seq - ) - .toArray(); - if (!rows.length) throw new Error(`pack ${packId}: missing chunk ${seq}`); - const chunk = new Uint8Array(rows[0].data); + let chunk: Uint8Array; + if (this.backendOf(packId) === "r2") { + if (!this.r2) throw new Error(`pack ${packId}: R2 backend not bound`); + const obj = await this.r2.bucket.get(this.rawKey(packId), { + range: { offset: seq * PACK_CHUNK, length: PACK_CHUNK }, + }); + if (!obj) throw new Error(`pack ${packId}: missing R2 object ${this.rawKey(packId)}`); + chunk = new Uint8Array(await obj.arrayBuffer()); + } else { + const rows = this.sql + .exec<{ data: ArrayBuffer }>( + "SELECT data FROM pack_data WHERE pack_id = ? AND seq = ?", + packId, + seq + ) + .toArray(); + if (!rows.length) throw new Error(`pack ${packId}: missing chunk ${seq}`); + chunk = new Uint8Array(rows[0].data); + } this.chunks.set(key, chunk); while (this.chunks.size > RAW_CHUNK_CACHE) { this.chunks.delete(this.chunks.keys().next().value as string); @@ -317,7 +418,7 @@ * then applied from the base upward, holding at most two inflated buffers at * a time. This caps both stack depth and peak memory regardless of input. */ - getObject(oid: string, cache: ObjCache): ObjRec | null { + async getObject(oid: string, cache: ObjCache): Promise<ObjRec | null> { const cached = cache.get(oid); if (cached) return cached; const first = this.lookup(oid); @@ -334,7 +435,7 @@ if (seen.has(cur.oid)) throw new Error(`cyclic delta chain at ${cur.oid}`); seen.add(cur.oid); if (!cur.baseOid) { - base = { type: cur.type, data: pako.inflate(this.readRaw(cur.packId, cur.dataOff, cur.dataLen)) }; + base = { type: cur.type, data: inflateAll(await this.readRaw(cur.packId, cur.dataOff, cur.dataLen)) }; break; } if (chain.length >= MAX_DELTA_DEPTH) throw new Error(`delta chain exceeds depth ${MAX_DELTA_DEPTH} at ${oid}`); @@ -355,7 +456,7 @@ let obj = base; for (let i = chain.length - 1; i >= 0; i--) { const d = chain[i]; - const delta = pako.inflate(this.readRaw(d.packId, d.dataOff, d.dataLen)); + const delta = inflateAll(await this.readRaw(d.packId, d.dataOff, d.dataLen)); obj = { type: base.type, data: applyDelta(obj.data, delta) }; cache.put(d.oid, obj); } @@ -364,8 +465,9 @@ } /** - * Ingest a packfile arriving as a byte stream: store chunks verbatim - * (verifying the SHA-1 trailer on the fly), then index sequentially. + * Ingest a packfile arriving as a byte stream: store the bytes verbatim + * (to R2 when a bucket is bound, else SQLite chunk rows) while verifying the + * SHA-1 trailer on the fly, then index sequentially. * Deltas resolve eagerly against the LRU cache — pack ordering keeps * bases hot — so the straggler pass afterwards is nearly empty. Memory * stays bounded regardless of pack size. @@ -380,13 +482,24 @@ /** called periodically so the runtime can flush its write buffer — * workerd holds a request's dirty pages in the 128MB isolate heap */ flush?: () => Promise<void>; + /** screen every block with SHA-1DC (git-parity collision detection); + * default hashes with native crypto.subtle, which yields the identical + * oid without the collision check */ + collisionDetect?: boolean; } ): Promise<{ packId: number; count: number }> { - // a failed or interrupted ingest leaves rows without a pack_meta entry; - // reclaim that space before starting (pack_meta is only written on success) + const cd = opts.collisionDetect ?? false; + // a failed or interrupted ingest leaves rows (or an R2 raw object) without a + // pack_meta entry; reclaim that space before starting (pack_meta is only + // written on success). Union across every per-pack table so an R2 ingest that + // died mid-index — whose bytes are in R2, not pack_data — is caught too. const orphans = this.sql .exec<{ pack_id: number }>( - "SELECT DISTINCT d.pack_id AS pack_id FROM pack_data d LEFT JOIN pack_meta m ON m.pack_id = d.pack_id WHERE m.pack_id IS NULL" + `SELECT pack_id FROM ( + SELECT DISTINCT pack_id FROM pack_data + UNION SELECT DISTINCT pack_id FROM pack_objects + UNION SELECT DISTINCT pack_id FROM pack_pending + ) WHERE pack_id NOT IN (SELECT pack_id FROM pack_meta)` ) .toArray(); for (const o of orphans) { @@ -393,6 +506,14 @@ for (const t of ["pack_data", "pack_objects", "pack_pending"]) { this.sql.exec(`DELETE FROM ${t} WHERE pack_id = ?`, o.pack_id); } + // an interrupted R2 ingest may have left a completed raw object; delete it + if (this.r2) { + try { + await this.r2.bucket.delete(this.rawKey(o.pack_id)); + } catch { + // best effort: the packId is reused next and its raw key overwritten + } + } } // the reclaimed pack id is about to be handed out again: drop anything the // chunk LRU still holds for it (and for anything else — it is only a cache) @@ -405,7 +526,20 @@ console.log(`[ingest pack ${packId} +${Math.round((Date.now() - started) / 1000)}s] ${msg.trim()}`); }; - // phase A: stream bytes into chunk rows, hashing all but the trailing 20. + // storage backend: with an R2 binding the pack bytes stream to + // raw/<repo>/<packId> via multipart and only the index stays in SQLite; else + // they go to pack_data as before. Prime the backend cache now — before + // pack_meta exists — so phase B/C readRaw hits the right source. A refused + // multipart begin (celld makes createMultipartUpload throw) degrades to SQLite. + let capture: MultipartCapture | null = null; + if (this.r2) { + const mp = await beginRawMultipart(this.r2.bucket, this.rawKey(packId)); + if (mp) capture = new MultipartCapture(mp); + } + const r2Backed = capture !== null; + this.backend.set(packId, r2Backed ? "r2" : "sql"); + + // phase A: stream bytes into R2 (or chunk rows), hashing all but the trailing 20. // Strictly linear: a cursor fills a fixed 1MB buffer — the source may // arrive as one multi-GB chunk (celld buffers request bodies), so any // re-concatenation of the remainder would go quadratic. @@ -430,6 +564,10 @@ } else { tail = joined.slice(); } + if (capture) { + capture.push(data); // copied into a 16 MiB multipart part buffer + return; + } let off = 0; while (off < data.length) { const take = Math.min(PACK_CHUNK - chunkLen, data.length - off); @@ -453,6 +591,10 @@ const feedSliced = async (data: Uint8Array) => { for (let off = 0; off < data.length; off += FEED_SLICE) { feed(data.subarray(off, off + FEED_SLICE)); + if (capture) { + await capture.drain(); // upload whole 16 MiB parts, keep only the remainder + if (capture.aborted) throw new Error("pack upload to R2 failed"); + } if (data.length > FEED_SLICE) await new Promise((res) => setTimeout(res, 0)); await opts.flush?.(); } @@ -465,7 +607,7 @@ if (value?.length) await feedSliced(value); } } - if (chunkLen > 0) { + if (!capture && chunkLen > 0) { this.sql.exec( "INSERT INTO pack_data (pack_id, seq, data) VALUES (?, ?, ?)", packId, @@ -473,32 +615,59 @@ chunkBuf.slice(0, chunkLen).buffer ); } - if (total < 32) throw new Error("pack too small"); - if (toHex(tail) !== toHex(sha.digest())) throw new Error("pack checksum mismatch"); + // validate the trailer before completing the R2 upload, so a corrupt push + // never leaves a finished raw object (abort leaves R2 with nothing). + if (total < 32) { + if (capture) await capture.abort(); + throw new Error("pack too small"); + } + if (toHex(tail) !== toHex(sha.digest())) { + if (capture) await capture.abort(); + throw new Error("pack checksum mismatch"); + } + if (capture) { + // complete the multipart BEFORE phase B/C: both readRaw the finished raw + // object (phase B for eager delta bases, phase C for deferred deltas). + await capture.finish(); + if (capture.aborted) throw new Error("pack upload to R2 failed"); + } say(`Received pack: ${Math.round(total / 1048576)}MB\n`); // phase B: sequential scan with eager delta resolution - const r = new PackReader(this.sql, packId, total); + const r = new PackReader((s) => this.chunk(packId, s), total); const magic = new Uint8Array(4); - for (let i = 0; i < 4; i++) magic[i] = r.byte(); + for (let i = 0; i < 4; i++) magic[i] = await r.byte(); if (td.decode(magic) !== "PACK") throw new Error("bad pack signature"); let version = 0, count = 0; - for (let i = 0; i < 4; i++) version = (version << 8) | r.byte(); - for (let i = 0; i < 4; i++) count = (count << 8) | r.byte(); + for (let i = 0; i < 4; i++) version = (version << 8) | (await r.byte()); + for (let i = 0; i < 4; i++) count = (count << 8) | (await r.byte()); if (version !== 2 && version !== 3) throw new Error(`unsupported pack version ${version}`); - // single-row statements: celld's SQLite caps bound variables far lower - // than workerd, and in-process inserts are cheap even at Linux scale - const insertObject = (row: (string | number | null)[]) => { - // OR IGNORE, never OR REPLACE: a duplicate object must keep its FIRST - // location. Re-pointing it at the incoming pack means the orphan sweep - // erases it from the index if this ingest fails — objects that were - // safely stored become unresolvable ("missing base") forever after. + // batched index inserts: one exec per object made pack_objects a top SQL + // cost, and deferring rows is safe because delta bases resolve from the + // in-memory offset window and object cache — the flush points below cover + // the rare pack_objects SELECT fallbacks (oidByOffset miss, getObject). + // OR IGNORE keeps a duplicate oid's FIRST location — including within one + // batch — so a failed ingest's orphan sweep can't strand a stored object. + // workerd caps a DO SQL statement at ~100 bound variables (celld's stock + // SQLite allows more); 10 rows x 9 cols = 90 stays under that on both. + const OBJ_BATCH = 10; + let objBatch: (string | number | null)[] = []; + let objBatchRows = 0; + const flushObjects = () => { + if (!objBatchRows) return; + const values = Array(objBatchRows).fill("(?,?,?,?,?,?,?,?,?)").join(","); this.sql.exec( - "INSERT OR IGNORE INTO pack_objects (oid, pack_id, offset, data_off, data_len, type, size, entry_size, base_oid) VALUES (?,?,?,?,?,?,?,?,?)", - ...row + `INSERT OR IGNORE INTO pack_objects (oid, pack_id, offset, data_off, data_len, type, size, entry_size, base_oid) VALUES ${values}`, + ...objBatch ); + objBatch = []; + objBatchRows = 0; }; + const insertObject = (row: (string | number | null)[]) => { + for (const v of row) objBatch.push(v); + if (++objBatchRows >= OBJ_BATCH) flushObjects(); + }; const insertPending = (row: (string | number | null)[]) => { this.sql.exec( "INSERT INTO pack_pending (pack_id, offset, data_off, data_len, entry_size, base_oid, base_offset) VALUES (?,?,?,?,?,?,?)", @@ -519,6 +688,7 @@ const oidByOffset = (off: number): string | null => { const hit = winCur.get(off) ?? winPrev.get(off); if (hit) return hit; + flushObjects(); // a windowed miss falls back to SELECT: deferred rows must be visible const rows = this.sql .exec<{ oid: string }>("SELECT oid FROM pack_objects WHERE pack_id = ? AND offset = ?", packId, off) .toArray(); @@ -528,12 +698,12 @@ let pendingCount = 0; for (let i = 0; i < count; i++) { const offset = r.pos; - let byte = r.byte(); + let byte = await r.byte(); const type = (byte >> 4) & 7; let entrySize = byte & 15; let shift = 4; while (byte & 0x80) { - byte = r.byte(); + byte = await r.byte(); entrySize += (byte & 0x7f) * 2 ** shift; shift += 7; } @@ -540,16 +710,16 @@ let baseOid: string | null = null; let baseOffset: number | null = null; if (type === 6) { - byte = r.byte(); + byte = await r.byte(); let off = byte & 0x7f; while (byte & 0x80) { - byte = r.byte(); + byte = await r.byte(); off = (off + 1) * 128 + (byte & 0x7f); } baseOffset = offset - off; } else if (type === 7) { const b = new Uint8Array(20); - for (let j = 0; j < 20; j++) b[j] = r.byte(); + for (let j = 0; j < 20; j++) b[j] = await r.byte(); baseOid = toHex(b); } else if (!NUM_TYPE[type]) { throw new Error(`bad object type ${type} at ${offset}`); @@ -559,19 +729,24 @@ const isFull = type >= 1 && type <= 4; const objType = isFull ? NUM_TYPE[type] : null; const buffered = entrySize <= MAX_BUFFERED_ENTRY; - const pieces: Uint8Array[] = []; - if (isFull && !buffered) streamSha.reset().update(objectHeader(objType!, entrySize)); + // buffered entries inflate straight into one exact-size buffer (no piece + // array + concat, which would hold the object twice at once) + const content = buffered ? new Uint8Array(entrySize) : null; + if (isFull && !buffered) { + if (cd) streamSha.reset().update(objectHeader(objType!, entrySize)); + else streamShaPlain.reset().update(objectHeader(objType!, entrySize)); + } let inflated = 0; - const inf = new pako.Inflate(); + const inf = new pako.Inflate({ chunkSize: INFLATE_CHUNK }); (inf as unknown as { onData: (c: Uint8Array) => void }).onData = (c: Uint8Array) => { + if (content) content.set(c, inflated); + else if (isFull) (cd ? streamSha : streamShaPlain).update(c); inflated += c.length; - if (buffered) pieces.push(c); - else if (isFull) streamSha.update(c); }; const anyInf = inf as unknown as { err: number; msg: string; ended: boolean; strm: { avail_in: number } }; while (!anyInf.ended) { if (r.pos >= total) throw new Error(`truncated pack entry at ${offset}`); - const win = r.window(); + const win = await r.window(); inf.push(win, false); if (anyInf.err) throw new Error(`inflate failed at ${dataOff}: ${anyInf.msg}`); r.pos += anyInf.ended ? win.length - anyInf.strm.avail_in : win.length; @@ -580,14 +755,11 @@ const dataLen = r.pos - dataOff; if (isFull) { - let content: Uint8Array | null = null; - let oid: string; - if (buffered) { - content = pieces.length === 1 ? pieces[0] : concat(pieces); - oid = await hashObjectAsync(objType!, content); - } else { - oid = finishOid(streamSha); - } + const oid = buffered + ? await hashObjectAsync(objType!, content!, cd) + : cd + ? finishOid(streamSha) + : toHex(streamShaPlain.digest()); insertObject([oid, packId, offset, dataOff, dataLen, objType!, entrySize, entrySize, null]); winPut(offset, oid); if (content && content.length <= CACHE_ENTRY_LIMIT) { @@ -599,8 +771,9 @@ if (buffered && resolvedBase) { let base: ObjRec | null = opts.cache.get(resolvedBase) ?? null; if (!base) { + flushObjects(); // getObject SELECTs pack_objects: deferred rows must be visible try { - base = this.getObject(resolvedBase, opts.cache); + base = await this.getObject(resolvedBase, opts.cache); } catch { base = null; } @@ -607,13 +780,12 @@ } if (!base) base = this.extern(resolvedBase); if (base) { - const delta = pieces.length === 1 ? pieces[0] : concat(pieces); - const content = applyDelta(base.data, delta); - const oid = await hashObjectAsync(base.type, content); - insertObject([oid, packId, offset, dataOff, dataLen, base.type, content.length, entrySize, resolvedBase]); + const data = applyDelta(base.data, content!); + const oid = await hashObjectAsync(base.type, data, cd); + insertObject([oid, packId, offset, dataOff, dataLen, base.type, data.length, entrySize, resolvedBase]); winPut(offset, oid); - if (content.length <= CACHE_ENTRY_LIMIT) { - opts.cache.put(oid, { type: base.type, data: content }); + if (data.length <= CACHE_ENTRY_LIMIT) { + opts.cache.put(oid, { type: base.type, data }); } resolved = true; } @@ -666,15 +838,16 @@ } if (!baseOid) continue; let base: ObjRec | null = null; + flushObjects(); // getObject SELECTs pack_objects: deferred rows must be visible try { - base = this.getObject(baseOid, opts.cache) ?? this.extern(baseOid); + base = (await this.getObject(baseOid, opts.cache)) ?? this.extern(baseOid); } catch { base = null; } if (!base) continue; // thin base not available (yet) - const delta = pako.inflate(this.readRaw(packId, row.data_off, row.data_len)); + const delta = inflateAll(await this.readRaw(packId, row.data_off, row.data_len)); const data = applyDelta(base.data, delta); - const oid = await hashObjectAsync(base.type, data); + const oid = await hashObjectAsync(base.type, data, cd); insertObject([oid, packId, row.offset, row.data_off, row.data_len, base.type, data.length, row.entry_size, baseOid]); this.sql.exec("DELETE FROM pack_pending WHERE pack_id = ? AND offset = ?", packId, row.offset); winPut(row.offset, oid); @@ -693,10 +866,11 @@ } } if (resolvedTotal) say(`Resolved ${resolvedTotal} deferred delta(s)\n`); + flushObjects(); // commit the final partial batch before pack_meta marks the pack complete this.sql.exec( - "INSERT INTO pack_meta (pack_id, size, count, created) VALUES (?, ?, ?, ?)", - packId, total, count, Date.now() + "INSERT INTO pack_meta (pack_id, size, count, created, store) VALUES (?, ?, ?, ?, ?)", + packId, total, count, Date.now(), r2Backed ? "r2" : "sql" ); return { packId, count }; } diff --git a/src/git/protocol.ts b/src/git/protocol.ts @@ -3,6 +3,7 @@ import { GitStore } from "./store"; import { PackWriter } from "./pack"; import { OidSet } from "./oidset"; +import { MultipartCapture, MultipartPackUpload } from "./multipart"; import { parseCommit, parseTag, parseTree, isGitlinkMode, isTreeMode, TYPE_NUM, NUM_TYPE } from "./objects"; const AGENT = "agent=dgit/0.3"; @@ -20,6 +21,20 @@ export type Service = "git-upload-pack" | "git-receive-pack"; +/** + * Optional R2-backed full-clone offload, write side only: on a full-clone MISS + * the DO tees the generated pack into a multipart upload so it never buffers the + * whole pack in isolate memory (no size cap). Absent on celld and on Workers + * without the binding, in which case every clone is served from the DO exactly + * as before. HITs are served by the Worker streaming R2 directly, not the DO. + */ +export interface PackCache { + repo: string; + /** Begin a multipart pack build under `key`, stamping the object count as + * customMetadata; null when R2 is unavailable (the clone still streams). */ + beginMultipart(key: string, objects: number): Promise<MultipartPackUpload | null>; +} + /** GET /info/refs?service=... — smart ref advertisement (protocol v0). */ export function advertisement(store: GitStore, service: Service): Uint8Array { const caps = @@ -64,7 +79,7 @@ * who learned its sha. Membership in a set of ref tips, so a normal fetch * (which only ever wants advertised tips) pays a few hash lookups, not a walk. */ -export function advertisedOids(store: GitStore): Set<string> { +export async function advertisedOids(store: GitStore): Promise<Set<string>> { const oids = new Set<string>(); const head = store.resolveHead(); if (head) oids.add(head); @@ -73,7 +88,7 @@ // peel annotated tags; lightweight tags already point at the commit if (!r.name.startsWith("refs/tags/")) continue; if (store.typeAndSize(r.target)?.type !== "tag") continue; - const peeled = peelToCommitOid(store, r.target); + const peeled = await peelToCommitOid(store, r.target); if (peeled) oids.add(peeled); } return oids; @@ -85,7 +100,7 @@ * cache describe precisely what the client asked for; a subset of the tips would * pull unrelated branches' objects in and leave the client with danglers. */ -function wantsAreAllTips(store: GitStore, wants: string[]): boolean { +export function wantsAreAllTips(store: GitStore, wants: string[]): boolean { const tips = new Set<string>(); for (const r of store.refs()) tips.add(r.target); if (tips.size === 0) return false; @@ -107,6 +122,57 @@ return frames; } +/** + * Serve a full clone by streaming the R2-cached raw pack straight from the + * Worker: the NAK preamble a full-clone negotiation always produces, the + * optional band-2 progress line, then the pack re-chunked into band-1 side-band + * frames as R2 hands it over, then flush. Framing is identical to the DO path + * and the pack bytes are the stored bytes, so the client receives a + * byte-for-byte-equal pack. Memory is one R2 read chunk at a time — never + * proportional to pack size — and the DO is never contacted for bytes. + */ +export function streamingCloneResponse( + body: ReadableStream<Uint8Array>, + objects: number, + noProgress: boolean +): Response { + const headers = { + "content-type": "application/x-git-upload-pack-result", + "cache-control": "no-cache", + "x-pack-cache": "hit", + }; + const reader = body.getReader(); + let started = false; + let done = false; + const stream = new ReadableStream<Uint8Array>({ + pull: async (ctrl) => { + try { + if (!started) { + started = true; + ctrl.enqueue(pkt("NAK\n")); + if (!noProgress) { + for (const f of sidebandFrames(2, te.encode(`Enumerating objects: ${objects}, done.\n`))) ctrl.enqueue(f); + } + } + const { done: rdone, value } = await reader.read(); + if (rdone) { + if (!done) { + done = true; + ctrl.enqueue(FLUSH); + ctrl.close(); + } + return; + } + for (const f of sidebandFrames(1, value)) ctrl.enqueue(f); + } catch (err) { + ctrl.error(err); + } + }, + cancel: () => reader.cancel(), + }); + return new Response(stream, { headers }); +} + interface UploadRequest { wants: string[]; haves: string[]; @@ -116,7 +182,7 @@ caps: Set<string>; } -function parseUploadRequest(body: Uint8Array): UploadRequest { +export function parseUploadRequest(body: Uint8Array): UploadRequest { const parser = new PktParser(body); const req: UploadRequest = { wants: [], @@ -147,9 +213,9 @@ } /** Follow tag objects down to the underlying commit oid (or null). */ -function peelToCommitOid(store: GitStore, oid: string): string | null { +async function peelToCommitOid(store: GitStore, oid: string): Promise<string | null> { for (let i = 0; i < 10; i++) { - const obj = store.get(oid); + const obj = await store.get(oid); if (!obj) return null; if (obj.type === "commit") return oid; if (obj.type === "tag") { @@ -165,16 +231,16 @@ * Depth-limited commit set from the wants (BFS, min depth wins; the tip is * depth 1, like git). Boundary commits are included but their parents cut. */ -function computeDepthSet( +async function computeDepthSet( store: GitStore, wants: string[], depth: number -): { commits: Set<string>; boundary: Set<string> } { +): Promise<{ commits: Set<string>; boundary: Set<string> }> { const commits = new Set<string>(); const boundary = new Set<string>(); const queue: { oid: string; depth: number }[] = []; for (const w of wants) { - const c = peelToCommitOid(store, w); + const c = await peelToCommitOid(store, w); if (c && !commits.has(c)) { commits.add(c); queue.push({ oid: c, depth: 1 }); @@ -182,7 +248,7 @@ } while (queue.length) { const { oid, depth: d } = queue.shift()!; - const obj = store.get(oid); + const obj = await store.get(oid); if (obj?.type !== "commit") continue; const parents = parseCommit(obj.data).parents; if (d >= depth) { @@ -221,13 +287,13 @@ const set = new OidSet(4096); const stack: string[] = []; for (const w of wants) { - const c = peelToCommitOid(store, w); + const c = await peelToCommitOid(store, w); if (c && !haveCommits.has(c) && set.addHex(c)) stack.push(c); } let minTime = Infinity; while (stack.length) { const oid = stack.pop()!; - const obj = store.get(oid); + const obj = await store.get(oid); if (obj?.type !== "commit") continue; await yieldMaybe(); const c = parseCommit(obj.data); @@ -272,7 +338,7 @@ const haveCommits = new Set<string>(); for (const h of haves) { if (!store.has(h)) continue; - const c = peelToCommitOid(store, h); + const c = await peelToCommitOid(store, h); if (c) haveCommits.add(c); } @@ -289,7 +355,7 @@ } while (commitStack.length) { const oid = commitStack.pop()!; - const obj = store.get(oid); + const obj = await store.get(oid); if (!obj) continue; await yieldMaybe(); if (obj.type === "tag") { @@ -314,7 +380,7 @@ for (let i = 0; i < interesting.size; i++) { const oid = interesting.atHex(i); if (excluded.hasHex(oid)) continue; // contested: reachable from a have too - const obj = store.get(oid); + const obj = await store.get(oid); if (obj?.type !== "commit") continue; await yieldMaybe(); for (const p of parseCommit(obj.data).parents) { @@ -324,7 +390,7 @@ const trees: string[] = []; for (const oid of boundary) { - const obj = store.get(oid); + const obj = await store.get(oid); if (obj?.type !== "commit") continue; const tree = parseCommit(obj.data).tree; if (tree && excluded.addHex(tree)) trees.push(tree); @@ -331,7 +397,7 @@ } while (trees.length) { const oid = trees.pop()!; - const obj = store.get(oid); + const obj = await store.get(oid); if (obj?.type !== "tree") continue; await yieldMaybe(); for (const e of parseTree(obj.data)) { @@ -377,7 +443,7 @@ if (meta.type === "commit" && commitLimit && !commitLimit.has(oid)) continue; if (excluded.hasHex(oid)) { if (descendThroughExcluded && meta.type === "commit") { - const full = store.get(oid)!; + const full = (await store.get(oid))!; stack.push(...parseCommit(full.data).parents); } continue; @@ -384,7 +450,7 @@ } walk.markHex(oid); if (meta.type === "blob") continue; // leaf: membership only - const full = store.get(oid)!; + const full = (await store.get(oid))!; if (full.type === "commit") { const c = parseCommit(full.data); stack.push(c.tree, ...c.parents); @@ -409,7 +475,8 @@ export async function uploadPack( store: GitStore, body: Uint8Array, - release: () => void = () => {} + release: () => void = () => {}, + packCache?: PackCache ): Promise<Response> { const headers = { "content-type": "application/x-git-upload-pack-result", @@ -421,7 +488,7 @@ return new Response(pkt("ERR no valid wants\n") as unknown as BodyInit, { headers }); } // a want must be an object we currently advertise, not merely one we store - const allowed = advertisedOids(store); + const allowed = await advertisedOids(store); for (const w of req.wants) { if (!allowed.has(w) || !store.has(w)) { release(); @@ -441,7 +508,7 @@ if (store.has(s)) preamble.push(pkt(`unshallow ${s}\n`)); } } else { - const { commits, boundary } = computeDepthSet(store, req.wants, req.deepen); + const { commits, boundary } = await computeDepthSet(store, req.wants, req.deepen); commitLimit = commits; for (const b of boundary) { if (!clientShallowSet.has(b)) preamble.push(pkt(`shallow ${b}\n`)); @@ -496,6 +563,9 @@ return new Response(concat(preamble) as unknown as BodyInit, { headers }); } + const sideband = req.caps.has("side-band-64k"); + const noProgress = req.caps.has("no-progress"); + // Full-clone fast path: no haves, not shallow/deepen, and the wants are // exactly the current ref tips. The reachable object set is then identical to // what the graph walk would rediscover, so serve it from the versioned cache @@ -504,6 +574,18 @@ // through to the walk, which then repopulates it — so a wrong pack is never // served, and existing repos become fast on their next full clone. const fullClone = !req.haves.length && req.deepen === 0 && wantsAreAllTips(store, req.wants); + + // R2 full-clone offload (write side). Only a standard side-band-64k full clone + // is eligible, and it is reached only after the advertised-oids gate above — so + // the versioned key can never warm an unreachable/force-pushed object. The key + // is the ref version, so a push/force-push/delete lands on a fresh key and + // stale packs are simply never referenced. HITs are served by the Worker + // streaming R2 directly; the DO only reaches here on a MISS, where it serves + // the client and self-warms the cache via multipart while it streams (see + // `capture` below). Absent R2 (celld / no binding) this is a plain DO clone. + const r2Key = + fullClone && sideband && packCache ? `pack/${packCache.repo}/${store.reachableVersion()}` : null; + let send = fullClone ? store.loadReachable() : null; if (!send) { const excluded = await excludedObjects(store, req.haves, req.clientShallows, req.wants); @@ -621,9 +703,6 @@ }); } - const sideband = req.caps.has("side-band-64k"); - const noProgress = req.caps.has("no-progress"); - // stream the pack: preamble, then pack bytes (side-band framed if negotiated) const pending: Uint8Array[] = [concat(preamble)]; let buffered: Uint8Array[] = []; @@ -636,7 +715,18 @@ if (sideband) pending.push(...sidebandFrames(1, payload)); else pending.push(payload); }; + // Self-warm R2 on a full-clone MISS: tee the raw pack (writer output, verbatim + // — same bytes a later HIT re-frames) into a multipart upload alongside the + // live stream. The pack streams into R2 in 16 MiB parts as it is generated, so + // isolate memory is never proportional to pack size and there is no size cap. + // A failed begin just streams the clone without caching. + let capture: MultipartCapture | null = null; + if (r2Key && packCache) { + const mp = await packCache.beginMultipart(r2Key, total); + if (mp) capture = new MultipartCapture(mp); + } const writer = new PackWriter((chunk) => { + if (capture) capture.push(chunk); buffered.push(chunk); bufferedLen += chunk.length; if (bufferedLen >= SIDEBAND_CHUNK) flushBuffered(); @@ -653,7 +743,7 @@ for (const c of pending) ctrl.enqueue(c); pending.length = 0; }, - pull: (ctrl) => { + pull: async (ctrl) => { try { for (let n = 0; n < 64 && i < total; n++, i++) { const idx = order ? order[i] : i; @@ -663,11 +753,11 @@ const oid = send.markedAtHex(idx); const entry = store.packs.lookup(oid); if (entry && !entry.baseOid) { - writer.rawFull(entry.type, entry.entrySize, store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen)); + writer.rawFull(entry.type, entry.entrySize, await store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen)); } else if (entry && entry.baseOid && send.isMarkedHex(entry.baseOid)) { - writer.rawDelta(entry.entrySize, entry.baseOid, store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen)); + writer.rawDelta(entry.entrySize, entry.baseOid, await store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen)); } else { - const obj = store.get(oid); + const obj = await store.get(oid); if (!obj) throw new Error(`missing object ${oid}`); writer.object(obj.type, obj.data); } @@ -677,12 +767,12 @@ // the lean typed arrays, so no SQL lookup happens here. const code = eCode[idx]; if (code <= -3) { - writer.rawFull(NUM_TYPE[-2 - code], eSize![idx], store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx])); + writer.rawFull(NUM_TYPE[-2 - code], eSize![idx], await store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx])); } else if (code >= 0) { - writer.rawDelta(eSize![idx], send.atHex(code), store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx])); + writer.rawDelta(eSize![idx], send.atHex(code), await store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx])); } else { const oid = send.markedAtHex(idx); - const obj = store.get(oid); + const obj = await store.get(oid); if (!obj) throw new Error(`missing object ${oid}`); writer.object(obj.type, obj.data); } @@ -697,17 +787,28 @@ } for (const c of pending) ctrl.enqueue(c); pending.length = 0; + // Warm R2 in lock-step with generation: the client already has this + // batch's bytes, and awaiting the part upload here is what bounds isolate + // memory to one part regardless of pack size. finish() flushes the final + // part and completes; a failure aborts, leaving no partial R2 object. + if (capture) { + if (finished) await capture.finish(); + else await capture.drain(); + } if (finished) { ctrl.close(); release(); // stream fully drained: this upload no longer holds a slot } } catch (err) { + if (capture) await capture.abort(); release(); ctrl.error(err); } }, - cancel: () => { - // client went away mid-transfer (e.g. negotiation round abort) + cancel: async () => { + // client went away mid-transfer (e.g. negotiation round abort): abort the + // half-built multipart upload so R2 keeps no partial object + if (capture) await capture.abort(); release(); }, }); @@ -715,13 +816,13 @@ } /** Is `anc` an ancestor of (or equal to) `desc`? Bounded walk. */ -export function isAncestor(store: GitStore, anc: string, desc: string): boolean { +export async function isAncestor(store: GitStore, anc: string, desc: string): Promise<boolean> { if (anc === desc) return true; const seen = new Set<string>([desc]); const stack = [desc]; let visited = 0; while (stack.length && visited++ < 10000) { - const obj = store.get(stack.pop()!); + const obj = await store.get(stack.pop()!); if (obj?.type !== "commit") continue; for (const p of parseCommit(obj.data).parents) { if (p === anc) return true; @@ -778,7 +879,7 @@ * closure was validated at their own push, so descent stops there — the walk * touches only objects this push introduced. */ -function reachableComplete(store: GitStore, tip: string, known: OidSet): boolean { +async function reachableComplete(store: GitStore, tip: string, known: OidSet): Promise<boolean> { const stack = [tip]; while (stack.length) { const oid = stack.pop()!; @@ -786,7 +887,7 @@ const meta = store.typeAndSize(oid); if (!meta) return false; if (meta.type === "blob") continue; - const obj = store.get(oid); + const obj = await store.get(oid); if (!obj) return false; if (obj.type === "commit") { const c = parseCommit(obj.data); @@ -803,15 +904,38 @@ return true; } -/** Apply ref updates after the pack (if any) has been ingested. */ -export function applyPushCommands( +/** Per-command outcome decided by the object-reading validation pass. */ +interface PushDecision { + ref: string; + expectedOld: string; // CAS value re-checked atomically at commit + reject?: string; // ng message; the command does not apply + del?: boolean; // delete the ref + set?: string; // move the ref to this oid + forced?: boolean; // a set that strands old history (non-fast-forward) +} + +/** Result of the async push validation, consumed by the sync commit under a savepoint. */ +export interface PushPlan { + decisions: PushDecision[]; + known: OidSet; + hadCache: boolean; +} + +/** + * Validate push commands against the (already ingested) object database. This + * is the object-reading half — connectivity + ancestry walks — and is async + * because an R2-backed pack's bytes are fetched over the network. It performs no + * writes; the ref mutations happen in commitPush under a savepoint. Splitting + * the read pass out is what lets the writes stay inside transactionSync (which + * cannot await) while still reading R2-backed objects. Pushes are serialized + * (receiveChain) and only pushes write refs, so ref state read here is stable + * through to commit; commitPush re-checks each CAS atomically regardless. + */ +export async function validatePush( store: GitStore, commands: PushCommand[], unpackError: string | null -): { results: CommandResult[]; changed: boolean; needsGc: boolean } { - const results: CommandResult[] = []; - let changed = false; - let needsGc = false; +): Promise<PushPlan> { // whether the reachable cache is valid for the pre-push refs: only then can // it be extended in place (union with the connectivity walk) instead of // rebuilt from scratch on the next full clone @@ -821,14 +945,16 @@ const known = new OidSet(4096); for (const r of store.refs()) { known.addHex(r.target); - const c = peelToCommitOid(store, r.target); + const c = await peelToCommitOid(store, r.target); if (c) known.addHex(c); } const knownHead = store.resolveHead(); if (knownHead) known.addHex(knownHead); + + const decisions: PushDecision[] = []; for (const cmd of commands) { if (unpackError) { - results.push({ ref: cmd.ref, ok: false, msg: "unpacker error" }); + decisions.push({ ref: cmd.ref, expectedOld: cmd.old, reject: "unpacker error" }); continue; } const bad = !cmd.ref.startsWith("refs/") || cmd.ref.includes("..") || BAD_REF.test(cmd.ref) @@ -836,29 +962,61 @@ || cmd.ref.endsWith("/") || cmd.ref.length > 255 || cmd.ref.split("/").some((c) => c.startsWith(".")); if (bad) { - results.push({ ref: cmd.ref, ok: false, msg: "funny refname" }); + decisions.push({ ref: cmd.ref, expectedOld: cmd.old, reject: "funny refname" }); continue; } const current = store.getRef(cmd.ref) ?? ZERO_OID; if (current !== cmd.old) { - results.push({ ref: cmd.ref, ok: false, msg: "fetch first" }); + decisions.push({ ref: cmd.ref, expectedOld: cmd.old, reject: "fetch first" }); continue; } if (cmd.next === ZERO_OID) { - store.delRef(cmd.ref); + decisions.push({ ref: cmd.ref, expectedOld: cmd.old, del: true }); + continue; + } + if (!(await reachableComplete(store, cmd.next, known))) { + decisions.push({ ref: cmd.ref, expectedOld: cmd.old, reject: "missing necessary objects" }); + continue; + } + const forced = cmd.old !== ZERO_OID && !(await isAncestor(store, cmd.old, cmd.next)); + decisions.push({ ref: cmd.ref, expectedOld: cmd.old, set: cmd.next, forced }); + } + return { decisions, known, hadCache }; +} + +/** + * Apply a validated PushPlan under a savepoint (transactionSync): a multi-ref + * push must not leave half its branches moved if a later update throws. Each CAS + * is re-checked here against live ref state so the write is atomic with its + * guard; a mismatch (a ref moved between validation and commit, which the push + * serialization normally prevents) both rejects that command and forces the + * reachable cache to be rebuilt rather than extended from a now-stale walk. + */ +export function commitPush(store: GitStore, plan: PushPlan): { results: CommandResult[]; changed: boolean; needsGc: boolean } { + const results: CommandResult[] = []; + let changed = false; + let needsGc = false; + let casMismatch = false; + for (const d of plan.decisions) { + if (d.reject) { + results.push({ ref: d.ref, ok: false, msg: d.reject }); + continue; + } + const current = store.getRef(d.ref) ?? ZERO_OID; + if (current !== d.expectedOld) { + results.push({ ref: d.ref, ok: false, msg: "fetch first" }); + casMismatch = true; + continue; + } + if (d.del) { + store.delRef(d.ref); needsGc = true; } else { - if (!reachableComplete(store, cmd.next, known)) { - results.push({ ref: cmd.ref, ok: false, msg: "missing necessary objects" }); - continue; - } - if (cmd.old !== ZERO_OID && !isAncestor(store, cmd.old, cmd.next)) { - needsGc = true; // forced update strands old history - } - store.setRef(cmd.ref, cmd.next); + store.setRef(d.ref, d.set!); + if (d.forced) needsGc = true; // forced update strands old history } changed = true; - results.push({ ref: cmd.ref, ok: true }); + results.push({ ref: d.ref, ok: true }); } // keep HEAD pointing at a branch that exists (first push wins, prefer main/master) @@ -875,10 +1033,11 @@ // walk: the pre-existing tips plus every object this push newly connected. On // a pure fast-forward with a valid prior cache, union closes over exactly the // new reachable set. Anything else (a force-push/delete that strands history, - // or no prior cache to extend) invalidates it, so the next full clone rebuilds - // by walking rather than risk serving objects that are no longer reachable. + // no prior cache to extend, or a commit-time CAS mismatch that leaves `known` + // describing a command that did not apply) invalidates it, so the next full + // clone rebuilds by walking rather than risk serving unreachable objects. if (changed) { - if (hadCache && !needsGc) store.extendReachable(known); + if (plan.hadCache && !needsGc && !casMismatch) store.extendReachable(plan.known); else store.invalidateReachable(); } return { results, changed, needsGc }; diff --git a/src/git/store.ts b/src/git/store.ts @@ -103,10 +103,19 @@ return { type: rows[0].type, data: inflate(packed) }; } - get(oid: string): { type: ObjType; data: Uint8Array } | null { - return this.getLoose(oid) ?? this.packs.getObject(oid, this.cache); + async get(oid: string): Promise<{ type: ObjType; data: Uint8Array } | null> { + return this.getLoose(oid) ?? (await this.packs.getObject(oid, this.cache)); } + /** + * Bind (or clear) the R2 raw-pack backend for this repo. Absent binding keeps + * every pack in SQLite (celld / Workers without the binding). Idempotent and + * cheap: the DO is per-repo, so this is set on each request from the repo name. + */ + setR2(bucket: R2Bucket | undefined, repo: string): void { + this.packs.setR2(bucket, repo); + } + put(oid: string, type: ObjType, data: Uint8Array): void { if (this.sql.exec("SELECT 1 FROM objects WHERE oid = ?", oid).toArray().length > 0) return; const packed = deflate(data); diff --git a/src/index.ts b/src/index.ts @@ -2,7 +2,14 @@ import type { RepoInfo } from "./registry"; import { esc, age, layout, htmlResponse, errorPage } from "./ui/html"; import { CSS } from "./ui/style"; +import { parseUploadRequest, streamingCloneResponse } from "./git/protocol"; +import { isOid } from "./git/util"; +import { gunzipLimited } from "./git/zlib"; +import type { RepoCell } from "./repo"; +/** an upload-pack negotiation body is wants/haves/caps only — a few hundred KB */ +const MAX_UPLOAD_PACK_BYTES = 16 * 1024 * 1024; + export { RepoCell } from "./repo"; export { Registry } from "./registry"; @@ -193,6 +200,54 @@ } } +/** + * R2 full-clone fast path, served entirely from the Worker so the DO is never + * contacted for bytes (this is the read-offload that avoids the workerd + * DO->response stall on large packs). Detects a true full clone from the tiny + * negotiation body — wants, no haves, `done`, not shallow/deepen, side-band-64k + * — then asks the DO only for the cheap versioned key (no pack generation) and + * streams the stored raw pack from R2 with on-the-fly side-band framing. Returns + * null (a miss, a partial/shallow fetch, or any failure) to fall through to the + * normal DO clone path. Callers MUST invoke this only after the auth gate, so a + * private repo never streams R2 bytes to an anonymous client. + */ +async function serveCloneFromR2( + env: Env, + repo: string, + body: Uint8Array, + stub: DurableObjectStub<RepoCell> +): Promise<Response | null> { + const bucket = env.PACK_CACHE; + if (!bucket) return null; + const req = parseUploadRequest(body); + const fullClone = + req.wants.length > 0 && + req.wants.every(isOid) && + req.haves.length === 0 && + req.done && + req.deepen === 0 && + req.clientShallows.length === 0 && + req.caps.has("side-band-64k"); + if (!fullClone) return null; + let key: string | null; + try { + key = await stub.currentPackKey(repo, req.wants); + } catch { + return null; + } + if (!key) return null; + let obj: R2ObjectBody | null; + try { + obj = await bucket.get(key); + } catch { + return null; + } + if (!obj) return null; + const objects = parseInt(obj.customMetadata?.objects ?? "", 10); + if (!Number.isFinite(objects)) return null; + return streamingCloneResponse(obj.body, objects, req.caps.has("no-progress")); +} + export default { async fetch(req: Request, env: Env, ctx: ExecutionContext): Promise<Response> { const url = new URL(req.url); @@ -286,7 +341,30 @@ // MB binary packs). Forward explicit bytes there; stream on workerd. let fwdBody: BodyInit | null = null; if (req.method !== "GET" && req.method !== "HEAD" && req.body) { - fwdBody = pageCache() === null ? ((await req.arrayBuffer()) as ArrayBuffer) : req.body; + const declared = parseInt(req.headers.get("content-length") ?? "", 10); + const smallUploadPack = + sub === "/git-upload-pack" && (!Number.isFinite(declared) || declared <= MAX_UPLOAD_PACK_BYTES); + if (smallUploadPack) { + // buffer the tiny negotiation body so a full clone can be served straight + // from R2 (never touching the DO for bytes); the auth gate above has + // already run, so a private repo cannot reach this point anonymously. + const raw = new Uint8Array(await req.arrayBuffer()); + let parsed: Uint8Array = raw; + if (req.headers.get("content-encoding")?.includes("gzip")) { + try { + parsed = gunzipLimited(raw, MAX_UPLOAD_PACK_BYTES); + } catch { + parsed = new Uint8Array(0); + } + } + if (env.PACK_CACHE && parsed.length) { + const hit = await serveCloneFromR2(env, repo, parsed, stub); + if (hit) return hit; + } + fwdBody = raw; // miss: forward the original (still-encoded) bytes to the DO + } else { + fwdBody = pageCache() === null ? ((await req.arrayBuffer()) as ArrayBuffer) : req.body; + } } const fwd = new Request(doUrl.toString(), { method: req.method, diff --git a/src/repo.ts b/src/repo.ts @@ -9,10 +9,13 @@ advertisement, uploadPack, parsePushCommands, - applyPushCommands, + validatePush, + commitPush, renderStatus, sidebandFrames, + wantsAreAllTips, Service, + PackCache, } from "./git/protocol"; import { Commit, @@ -95,6 +98,16 @@ this.store = new GitStore(ctx.storage.sql); } + /** + * Bind the R2 raw-pack backend for this repo (absent binding keeps every pack + * in SQLite). The DO is per-repo, so the name is stable; persist it once so + * alarm-driven GC — which has no request to read x-repo from — can rebind. + */ + private bindR2(repo: string): void { + this.store.setR2(this.env.PACK_CACHE, repo); + if (this.env.PACK_CACHE && this.store.getMeta("repo") !== repo) this.store.setMeta("repo", repo); + } + async fetch(req: Request): Promise<Response> { const url = new URL(req.url); const repo = req.headers.get("x-repo") ?? "repo"; @@ -101,6 +114,7 @@ const host = req.headers.get("x-host") ?? url.host; const proto = req.headers.get("x-proto") ?? "https"; const path = url.pathname; + this.bindR2(repo); try { if (path === "/info/refs" && req.method === "GET") { @@ -136,7 +150,7 @@ this.activeUploads--; }; try { - return await uploadPack(this.store, await this.readBody(req), release); + return await uploadPack(this.store, await this.readBody(req), release, this.packCacheFor(repo)); } catch (err) { release(); // an over-inflating body is the client's fault, not a server error @@ -168,7 +182,12 @@ // this instance stays resident: bring the (empty) schema back so // later requests — including a re-creating push — find their tables this.store = new GitStore(this.ctx.storage.sql); + this.bindR2(repo); }); + // the R2 packs (clone cache + raw pack store) outlive the SQLite wipe; + // purge them so a re-created repo never inherits stale bytes (its + // versioned keys would differ anyway) + await this.purgePackCache(repo); return new Response("deleted\n"); } if (path === "/gc" && req.method === "POST") { @@ -176,7 +195,7 @@ } if (req.method !== "GET") return new Response("method not allowed\n", { status: 405 }); - return this.ui(repo, host, proto, path, url.searchParams); + return await this.ui(repo, host, proto, path, url.searchParams); } catch (err) { const msg = err instanceof Error ? err.message : String(err); return new Response(`error: ${msg}\n`, { status: 500 }); @@ -198,6 +217,9 @@ } // clear first: at-most-once, so a throwing sweep can't trigger a retry storm this.store.setMeta("gc-pending", "0"); + // no request here to carry x-repo: rebind R2 from the persisted repo name so + // the sweep can read R2-backed objects and drop their raw objects + this.bindR2(this.store.getMeta("repo") ?? "repo"); try { await this.gc(); } catch (err) { @@ -214,7 +236,7 @@ let result: { removed: number; kept: number; skipped?: boolean } = { removed: 0, kept: 0 }; const run = this.receiveChain.then(() => this.ctx.blockConcurrencyWhile(async () => { - result = this.runGc(); + result = await this.runGc(); }) ); this.receiveChain = run.then( @@ -248,7 +270,7 @@ * packs are kept (deleting mid-pack is impossible without a repack); * huge repos skip the sweep entirely — the walk isn't worth it. */ - private runGc(): { removed: number; kept: number; skipped?: boolean } { + private async runGc(): Promise<{ removed: number; kept: number; skipped?: boolean }> { if (this.store.objectCount() > 300_000) { return { removed: 0, kept: this.store.objectCount(), skipped: true }; } @@ -262,7 +284,7 @@ const meta = this.store.typeAndSize(oid); if (!meta) continue; if (meta.type === "blob") continue; // leaf: never inflate blob content here - const obj = this.store.get(oid); + const obj = await this.store.get(oid); if (!obj) continue; if (obj.type === "commit") { const c = parseCommit(obj.data); @@ -296,7 +318,7 @@ for (let i = 0; i < reachable.size; i++) { const oid = reachable.atHex(i); if (this.store.packs.lookup(oid)) { - const obj = this.store.get(oid); + const obj = await this.store.get(oid); if (obj) { this.store.put(oid, obj.type, obj.data); migrated++; @@ -304,7 +326,11 @@ } } removed += packCount - migrated; + // capture the R2-backed pack ids before the index is dropped, then delete + // their raw objects after reset so no orphaned bytes linger in R2 + const r2ids = this.store.packs.r2PackIds(); this.store.packs.reset(); + await this.store.packs.deleteR2Packs(r2ids); } // sweep loose objects (including anything just migrated) unreachable now for (const oid of this.store.allOids()) { @@ -316,6 +342,74 @@ return noHeadroom ? { removed, kept: reachable.size, skipped: true } : { removed, kept: reachable.size }; } + /** + * Cheap full-clone key probe for the Worker's R2 fast path: returns the + * versioned pack key iff `wants` is exactly the current ref tips (a true full + * clone). Reads ref state only — it never generates a pack. null (wants are + * not all tips) sends the Worker back to the normal DO clone path. The key is + * the ref version, so it can never name a stale/force-pushed pack. + */ + currentPackKey(repo: string, wants: string[]): string | null { + if (!wantsAreAllTips(this.store, wants)) return null; + return `pack/${repo}/${this.store.reachableVersion()}`; + } + + /** + * R2 full-clone offload adapter (write side), or undefined when no bucket is + * bound (celld and Workers without the binding both fall back to the pure DO + * path). A throwing createMultipartUpload (celld deliberately makes R2 throw) + * degrades to null so the clone still streams uncached; part-level failures + * are handled by MultipartCapture aborting the upload. + */ + private packCacheFor(repo: string): PackCache | undefined { + const bucket = this.env.PACK_CACHE; + if (!bucket) return undefined; + return { + repo, + beginMultipart: async (key, objects) => { + try { + const mp = await bucket.createMultipartUpload(key, { + customMetadata: { objects: String(objects) }, + }); + const parts: R2UploadedPart[] = []; + return { + uploadPart: async (data) => { + parts.push(await mp.uploadPart(parts.length + 1, data)); + }, + complete: async () => { + await mp.complete(parts); + }, + abort: async () => { + await mp.abort(); + }, + }; + } catch (err) { + console.log(`[r2 ${repo}] multipart begin failed: ${err instanceof Error ? err.message : String(err)}`); + return null; + } + }, + }; + } + + /** Purge every R2 object for a repo on delete: clone-cache packs (all versions) + * and raw pack-store objects. */ + private async purgePackCache(repo: string): Promise<void> { + const bucket = this.env.PACK_CACHE; + if (!bucket) return; + for (const prefix of [`pack/${repo}/`, `raw/${repo}/`]) { + try { + for (let cursor: string | undefined; ; ) { + const listed = await bucket.list({ prefix, cursor }); + if (listed.objects.length) await bucket.delete(listed.objects.map((o) => o.key)); + if (!listed.truncated) break; + cursor = listed.cursor; + } + } catch (err) { + console.log(`[r2 ${repo}] purge ${prefix} failed: ${err instanceof Error ? err.message : String(err)}`); + } + } + } + private async readBody(req: Request): Promise<Uint8Array> { let body: Uint8Array = new Uint8Array(await req.arrayBuffer()); if (body.length > MAX_UPLOAD_PACK_BYTES) throw new Error("request body exceeds maximum size"); @@ -433,6 +527,7 @@ cache: new ObjCache(budget), onProgress: progress, flush: () => this.ctx.storage.sync(), + collisionDetect: this.env.SHA1DC === "1", }); } catch (err) { unpackError = err instanceof Error ? err.message : String(err); @@ -445,10 +540,13 @@ } } - // one savepoint around every ref update: a multi-ref push (git push --all) - // must not leave half its branches moved if a later update throws + // validate connectivity/ancestry first (async: an R2-backed pack's objects + // are fetched over the network, which transactionSync cannot await), then + // apply the ref writes under one savepoint — a multi-ref push (git push + // --all) must not leave half its branches moved if a later update throws + const plan = await validatePush(this.store, commands, unpackError); const { results, changed, needsGc } = this.ctx.storage.transactionSync(() => - applyPushCommands(this.store, commands, unpackError) + commitPush(this.store, plan) ); if (changed) { this.store.setMeta("created", "1"); @@ -474,7 +572,7 @@ } - private base(repo: string, tab: string, ref?: string, formAction?: string): Omit<LayoutOpts, "body"> { + private async base(repo: string, tab: string, ref?: string, formAction?: string): Promise<Omit<LayoutOpts, "body">> { const branches = this.store .refs() .filter((r) => r.name.startsWith("refs/heads/")) @@ -489,13 +587,13 @@ sub: this.store.getMeta("description") || "[no description]", tab, ref: ref ?? headBranch, - hasAbout: this.findReadme() !== null, + hasAbout: (await this.findReadme()) !== null, branches, formAction: formAction ?? `/${encodeURIComponent(repo)}/`, }; } - private ui(repo: string, host: string, proto: string, path: string, q: URLSearchParams): Response { + private async ui(repo: string, host: string, proto: string, path: string, q: URLSearchParams): Promise<Response> { const h = q.get("h") ?? undefined; if (path === "/" || path === "") return this.summaryPage(repo, host, proto); if (path === "/about/" || path === "/about") return this.aboutPage(repo, h); @@ -520,7 +618,7 @@ if (path.startsWith("/plain")) return this.plainPage(h, decodePath(path.slice("/plain".length))); if (path.startsWith("/blame")) return this.blamePage(repo, h, decodePath(path.slice("/blame".length))); if (path.startsWith("/snapshot/")) return this.snapshotPage(repo, decodeURIComponent(path.slice("/snapshot/".length))); - return errorPage(this.base(repo, "summary"), `page not found: ${path}`); + return errorPage(await this.base(repo, "summary"), `page not found: ${path}`); } @@ -531,13 +629,13 @@ * brute-forced through the UI even after the protocol side stopped serving * unreachable objects. */ - private resolveServable(id: string): string | null { + private async resolveServable(id: string): Promise<string | null> { if (!isOid(id) || !this.store.has(id)) return null; - return this.reachableOid(id) ? id : null; + return (await this.reachableOid(id)) ? id : null; } /** Is `target` reachable from any current ref (or HEAD)? */ - private reachableOid(target: string): boolean { + private async reachableOid(target: string): Promise<boolean> { // the full-clone reachable-set cache is versioned against current refs // (null on any mismatch), so a hit here is exactly as correct as the walk // below and skips inflating/parsing the whole object graph @@ -554,7 +652,7 @@ const meta = this.store.typeAndSize(oid); if (!meta) continue; if (meta.type === "blob") continue; - const obj = this.store.get(oid); + const obj = await this.store.get(oid); if (!obj) continue; if (obj.type === "commit") { const c = parseCommit(obj.data); @@ -572,7 +670,7 @@ } /** Resolve ?h= (branch, tag, full ref, or oid) to an object id. */ - private resolveRef(h?: string): { refName: string | null; oid: string } | null { + private async resolveRef(h?: string): Promise<{ refName: string | null; oid: string } | null> { if (!h) { const oid = this.store.resolveHead(); return oid ? { refName: this.store.head(), oid } : null; @@ -581,7 +679,7 @@ const oid = this.store.getRef(cand); if (oid) return { refName: cand, oid }; } - const full = this.resolveServable(h); + const full = await this.resolveServable(h); if (full) return { refName: null, oid: full }; return null; } @@ -591,15 +689,15 @@ * content, so a start oid that actually peeled through a tag has its final * commit oid cached — a hit skips every intermediate tag object read. */ - private peelToCommit(oid: string): { oid: string; commit: Commit } | null { + private async peelToCommit(oid: string): Promise<{ oid: string; commit: Commit } | null> { const start = oid; const cached = this.store.getPeeled(start); if (cached !== null) { - const obj = this.store.get(cached); + const obj = await this.store.get(cached); return obj?.type === "commit" ? { oid: cached, commit: parseCommit(obj.data) } : null; } for (let i = 0; i < 10; i++) { - const obj = this.store.get(oid); + const obj = await this.store.get(oid); if (!obj) return null; if (obj.type === "commit") { if (oid !== start) this.store.setPeeled(start, oid); @@ -614,16 +712,16 @@ return null; } - private loadCommit(oid: string): Commit | null { - const obj = this.store.get(oid); + private async loadCommit(oid: string): Promise<Commit | null> { + const obj = await this.store.get(oid); return obj?.type === "commit" ? parseCommit(obj.data) : null; } /** oid of the entry at `path` in this commit's tree (any type), or null. */ - private pathOid(commit: Commit, path: string[]): string | null { + private async pathOid(commit: Commit, path: string[]): Promise<string | null> { let oid = commit.tree; for (const seg of path) { - const obj = this.store.get(oid); + const obj = await this.store.get(oid); if (obj?.type !== "tree") return null; const entry = parseTree(obj.data).find((e) => e.name === seg); if (!entry) return null; @@ -633,12 +731,12 @@ } /** pathOid keyed by the root tree it starts from: shared across a log walk. */ - private pathOidCached(treeOid: string, path: string[], memo: Map<string, string | null>): string | null { + private async pathOidCached(treeOid: string, path: string[], memo: Map<string, string | null>): Promise<string | null> { const hit = memo.get(treeOid); if (hit !== undefined) return hit; let oid: string | null = treeOid; for (const seg of path) { - const obj = this.store.get(oid); + const obj = await this.store.get(oid); if (obj?.type !== "tree") { oid = null; break; } const entry = parseTree(obj.data).find((e) => e.name === seg); if (!entry) { oid = null; break; } @@ -648,11 +746,11 @@ return oid; } - private matchesFilter(e: LogEntry, filter: LogFilter, memo: Map<string, string | null>): boolean { + private async matchesFilter(e: LogEntry, filter: LogFilter, memo: Map<string, string | null>): Promise<boolean> { if (filter.path && filter.path.length) { - const mine = this.pathOidCached(e.commit.tree, filter.path, memo); - const parentTree = e.commit.parents[0] ? this.loadCommit(e.commit.parents[0])?.tree : undefined; - const theirs = parentTree ? this.pathOidCached(parentTree, filter.path, memo) : null; + const mine = await this.pathOidCached(e.commit.tree, filter.path, memo); + const parentTree = e.commit.parents[0] ? (await this.loadCommit(e.commit.parents[0]))?.tree : undefined; + const theirs = parentTree ? await this.pathOidCached(parentTree, filter.path, memo) : null; if (mine === theirs) return false; } if (filter.q) { @@ -670,8 +768,8 @@ } /** Date-ordered commit walk (newest first) with optional filtering. */ - private walkLog(tip: string, skip: number, limit: number, filter: LogFilter = {}): { entries: LogEntry[]; more: boolean } { - const first = this.peelToCommit(tip); + private async walkLog(tip: string, skip: number, limit: number, filter: LogFilter = {}): Promise<{ entries: LogEntry[]; more: boolean }> { + const first = await this.peelToCommit(tip); if (!first) return { entries: [], more: false }; skip = Math.min(skip, MAX_LOG_SCAN); // a wild ofs must not drive the walk to its cap for an empty page const scanCap = filter.path?.length ? PATH_LOG_SCAN : MAX_LOG_SCAN; @@ -683,11 +781,11 @@ while (frontier.length && out.length < skip + limit + 1 && scanned++ < scanCap) { frontier.sort((a, b) => b.commit.committer.time - a.commit.committer.time); const cur = frontier.shift()!; - if (this.matchesFilter(cur, filter, memo)) out.push(cur); + if (await this.matchesFilter(cur, filter, memo)) out.push(cur); for (const p of cur.commit.parents) { if (seen.has(p)) continue; seen.add(p); - const c = this.loadCommit(p); + const c = await this.loadCommit(p); if (c) frontier.push({ oid: p, commit: c }); } } @@ -695,17 +793,17 @@ } /** First-parent history of a path (for blame), newest first, with blobs. */ - private pathHistory(tip: string, path: string[], cap: number, maxBytes: number): BlameHistoryEntry[] { + private async pathHistory(tip: string, path: string[], cap: number, maxBytes: number): Promise<BlameHistoryEntry[]> { const out: BlameHistoryEntry[] = []; - let cur = this.peelToCommit(tip); + let cur = await this.peelToCommit(tip); let steps = 0; let bytes = 0; while (cur && steps++ < MAX_LOG_SCAN && out.length < cap) { - const myOid = this.pathOid(cur.commit, path); - const parent = cur.commit.parents[0] ? this.peelToCommit(cur.commit.parents[0]) : null; - const parentOid = parent ? this.pathOid(parent.commit, path) : null; + const myOid = await this.pathOid(cur.commit, path); + const parent = cur.commit.parents[0] ? await this.peelToCommit(cur.commit.parents[0]) : null; + const parentOid = parent ? await this.pathOid(parent.commit, path) : null; if (myOid !== parentOid) { - const blob = myOid ? this.store.get(myOid) : null; + const blob = myOid ? await this.store.get(myOid) : null; const data = blob?.type === "blob" ? blob.data : null; if (data) { bytes += data.length; @@ -720,7 +818,7 @@ } /** Map oid -> decorations (branch/tag pointing at it). */ - private decorations(repo: string): Map<string, string> { + private async decorations(repo: string): Promise<Map<string, string>> { const map = new Map<string, string>(); const r = `/${encodeURIComponent(repo)}`; for (const ref of this.store.refs()) { @@ -729,7 +827,7 @@ if (ref.name.startsWith("refs/heads/")) { html = `<a class='branch-deco' href='${r}/log/?h=${encodeURIComponent(ref.name.slice(11))}'>${esc(ref.name.slice(11))}</a>`; } else if (ref.name.startsWith("refs/tags/")) { - const peeled = this.peelToCommit(ref.target); + const peeled = await this.peelToCommit(ref.target); if (peeled) target = peeled.oid; html = `<a class='tag-deco' href='${r}/tag/?h=${encodeURIComponent(ref.name.slice(10))}'>${esc(ref.name.slice(10))}</a>`; } else continue; @@ -738,13 +836,13 @@ return map; } - private lookupPath( + private async lookupPath( rootTree: string, path: string[] - ): { kind: "tree"; entries: TreeEntry[] } | { kind: "blob"; entry: TreeEntry } | null { + ): Promise<{ kind: "tree"; entries: TreeEntry[] } | { kind: "blob"; entry: TreeEntry } | null> { let treeOid = rootTree; for (let i = 0; i < path.length; i++) { - const obj = this.store.get(treeOid); + const obj = await this.store.get(treeOid); if (obj?.type !== "tree") return null; const entry = parseTree(obj.data).find((e) => e.name === path[i]); if (!entry) return null; @@ -754,17 +852,17 @@ if (!isTreeMode(entry.mode)) return null; treeOid = entry.oid; } - const obj = this.store.get(treeOid); + const obj = await this.store.get(treeOid); if (obj?.type !== "tree") return null; return { kind: "tree", entries: parseTree(obj.data) }; } - private findReadme(): { name: string; oid: string } | null { + private async findReadme(): Promise<{ name: string; oid: string } | null> { const head = this.store.resolveHead(); if (!head) return null; - const c = this.peelToCommit(head); + const c = await this.peelToCommit(head); if (!c) return null; - const root = this.store.get(c.commit.tree); + const root = await this.store.get(c.commit.tree); if (root?.type !== "tree") return null; const entries = parseTree(root.data); for (const want of README_NAMES) { @@ -774,22 +872,22 @@ return null; } - private flattenTree(treeOid: string, prefix: string, out: Map<string, { oid: string; mode: string }>): void { - const obj = this.store.get(treeOid); + private async flattenTree(treeOid: string, prefix: string, out: Map<string, { oid: string; mode: string }>): Promise<void> { + const obj = await this.store.get(treeOid); if (obj?.type !== "tree") return; for (const e of parseTree(obj.data)) { const p = prefix ? `${prefix}/${e.name}` : e.name; - if (isTreeMode(e.mode)) this.flattenTree(e.oid, p, out); + if (isTreeMode(e.mode)) await this.flattenTree(e.oid, p, out); else if (!isGitlinkMode(e.mode)) out.set(p, { oid: e.oid, mode: e.mode }); } } - private computeDiff(oldTree: string | null, newTree: string): { files: FileDiff[]; truncated: boolean } { + private async computeDiff(oldTree: string | null, newTree: string): Promise<{ files: FileDiff[]; truncated: boolean }> { const oldFiles = new Map<string, { oid: string; mode: string }>(); const newFiles = new Map<string, { oid: string; mode: string }>(); - if (oldTree) this.flattenTree(oldTree, "", oldFiles); - this.flattenTree(newTree, "", newFiles); + if (oldTree) await this.flattenTree(oldTree, "", oldFiles); + await this.flattenTree(newTree, "", newFiles); const paths = [...new Set([...oldFiles.keys(), ...newFiles.keys()])].sort(); const files: FileDiff[] = []; let truncated = false; @@ -801,8 +899,8 @@ truncated = true; break; } - const oldData = o ? this.store.get(o.oid)?.data ?? new Uint8Array(0) : new Uint8Array(0); - const newData = n ? this.store.get(n.oid)?.data ?? new Uint8Array(0) : new Uint8Array(0); + const oldData = o ? (await this.store.get(o.oid))?.data ?? new Uint8Array(0) : new Uint8Array(0); + const newData = n ? (await this.store.get(n.oid))?.data ?? new Uint8Array(0) : new Uint8Array(0); const fd: FileDiff = { path: p, o, n, kind: "text", ops: null, hunks: [], add: 0, del: 0 }; if (isBinary(oldData) || isBinary(newData)) { fd.kind = "binary"; @@ -897,11 +995,11 @@ } - private aboutPage(repo: string, h: string | undefined): Response { - const base = this.base(repo, "about", h, `/${encodeURIComponent(repo)}/about/`); - const readme = this.findReadme(); + private async aboutPage(repo: string, h: string | undefined): Promise<Response> { + const base = await this.base(repo, "about", h, `/${encodeURIComponent(repo)}/about/`); + const readme = await this.findReadme(); if (!readme) return errorPage(base, "no readme found"); - const obj = this.store.get(readme.oid); + const obj = await this.store.get(readme.oid); if (!obj) return errorPage(base, "missing readme blob"); const text = td.decode(obj.data); const lower = readme.name.toLowerCase(); @@ -912,8 +1010,8 @@ return htmlResponse(layout({ ...base, body })); } - private summaryPage(repo: string, host: string, proto: string): Response { - const base = this.base(repo, "summary"); + private async summaryPage(repo: string, host: string, proto: string): Promise<Response> { + const base = await this.base(repo, "summary"); const branches = this.store.refs().filter((r) => r.name.startsWith("refs/heads/")); const tags = this.store.refs().filter((r) => r.name.startsWith("refs/tags/")); const r = `/${encodeURIComponent(repo)}`; @@ -930,23 +1028,23 @@ ); } - const branchRows = branches + const branchRows = (await Promise.all(branches .slice(0, 10) - .map((b) => { + .map(async (b) => { const name = b.name.slice(11); - const c = this.peelToCommit(b.target); + const c = await this.peelToCommit(b.target); if (!c) return ""; return `<tr><td><a href='${r}/log/?h=${encodeURIComponent(name)}'>${esc(name)}</a></td>` + `<td><a href='${r}/commit/?id=${c.oid}'>${esc(c.commit.subject)}</a></td>` + `<td>${esc(c.commit.author.name)}</td><td>${age(c.commit.committer.time)}</td></tr>`; - }) + }))) .join(""); - const tagRows = tags + const tagRows = (await Promise.all(tags .slice(0, 10) - .map((t) => { + .map(async (t) => { const name = t.name.slice(10); - const obj = this.store.get(t.target); + const obj = await this.store.get(t.target); let when = 0; let target = t.target; if (obj?.type === "tag") { @@ -954,7 +1052,7 @@ when = tag.tagger?.time ?? 0; target = tag.object; } - const c = this.peelToCommit(target); + const c = await this.peelToCommit(target); if (c && !when) when = c.commit.committer.time; const snap = `<a href='${r}/snapshot/${encodeURIComponent(repo)}-${encodeURIComponent(name)}.tar.gz'>tar.gz</a> ` + `<a href='${r}/snapshot/${encodeURIComponent(repo)}-${encodeURIComponent(name)}.zip'>zip</a>`; @@ -961,12 +1059,12 @@ return `<tr><td><a href='${r}/tag/?h=${encodeURIComponent(name)}'>${esc(name)}</a></td>` + `<td><a href='${r}/commit/?id=${target}'>${esc(c?.commit.subject ?? "")}</a></td>` + `<td>${esc(c?.commit.author.name ?? "")}</td><td>${age(when)}</td><td class='snapshots'>${snap}</td></tr>`; - }) + }))) .join(""); const headOid = this.store.resolveHead(); - const recent = headOid ? this.walkLog(headOid, 0, 10).entries : []; - const deco = this.decorations(repo); + const recent = headOid ? (await this.walkLog(headOid, 0, 10)).entries : []; + const deco = await this.decorations(repo); const logRows = recent .map( (e) => @@ -991,18 +1089,18 @@ return htmlResponse(layout({ ...base, body })); } - private logPage(repo: string, h: string | undefined, ofs: number, filter: LogFilter): Response { + private async logPage(repo: string, h: string | undefined, ofs: number, filter: LogFilter): Promise<Response> { const r = `/${encodeURIComponent(repo)}`; - const base = this.base(repo, "log", h, `${r}/log/`); + const base = await this.base(repo, "log", h, `${r}/log/`); let tip = h; if (filter.qt === "range" && filter.q) { tip = filter.q; filter = {}; } - const rr = this.resolveRef(tip); + const rr = await this.resolveRef(tip); if (!rr) return errorPage(base, tip ? `bad ref: ${tip}` : "empty repository"); - const { entries, more } = this.walkLog(rr.oid, ofs, LOG_PAGE, filter); - const deco = this.decorations(repo); + const { entries, more } = await this.walkLog(rr.oid, ofs, LOG_PAGE, filter); + const deco = await this.decorations(repo); const rows = entries .map( (e) => @@ -1049,27 +1147,27 @@ return htmlResponse(layout({ ...base, body })); } - private refsPage(repo: string): Response { + private async refsPage(repo: string): Promise<Response> { const r = `/${encodeURIComponent(repo)}`; - const base = this.base(repo, "refs", undefined, `${r}/refs/`); + const base = await this.base(repo, "refs", undefined, `${r}/refs/`); const refs = this.store.refs(); const branches = refs.filter((x) => x.name.startsWith("refs/heads/")); const tags = refs.filter((x) => x.name.startsWith("refs/tags/")); - const branchRows = branches - .map((b) => { + const branchRows = (await Promise.all(branches + .map(async (b) => { const name = b.name.slice(11); - const c = this.peelToCommit(b.target); + const c = await this.peelToCommit(b.target); const snap = `<a href='${r}/snapshot/${encodeURIComponent(repo)}-${encodeURIComponent(name)}.tar.gz'>tar.gz</a> ` + `<a href='${r}/snapshot/${encodeURIComponent(repo)}-${encodeURIComponent(name)}.zip'>zip</a>`; return `<tr><td><a href='${r}/log/?h=${encodeURIComponent(name)}'>${esc(name)}</a></td>` + `<td class='sha1'><a href='${r}/commit/?id=${b.target}'>${b.target.slice(0, 10)}</a></td>` + `<td>${esc(c?.commit.author.name ?? "")}</td><td>${c ? age(c.commit.committer.time) : ""}</td><td class='snapshots'>${snap}</td></tr>`; - }) + }))) .join(""); - const tagRows = tags - .map((t) => { + const tagRows = (await Promise.all(tags + .map(async (t) => { const name = t.name.slice(10); - const obj = this.store.get(t.target); + const obj = await this.store.get(t.target); let when = 0; let who = ""; let target = t.target; @@ -1088,7 +1186,7 @@ return `<tr><td><a href='${r}/tag/?h=${encodeURIComponent(name)}'>${esc(name)}</a></td>` + `<td class='sha1'><a href='${r}/commit/?id=${target}'>${target.slice(0, 10)}</a></td>` + `<td>${esc(who)}</td><td>${age(when)}</td><td class='snapshots'>${snap}</td></tr>`; - }) + }))) .join(""); const body = ` <table class='list nowrap'> @@ -1112,14 +1210,14 @@ return html; } - private treePage(repo: string, h: string | undefined, path: string[]): Response { + private async treePage(repo: string, h: string | undefined, path: string[]): Promise<Response> { const r = `/${encodeURIComponent(repo)}`; - const base = this.base(repo, "tree", h, `${r}/tree/${path.map(encodeURIComponent).join("/")}`); - const rr = this.resolveRef(h); + const base = await this.base(repo, "tree", h, `${r}/tree/${path.map(encodeURIComponent).join("/")}`); + const rr = await this.resolveRef(h); if (!rr) return errorPage(base, h ? `bad ref: ${h}` : "empty repository"); - const head = this.peelToCommit(rr.oid); + const head = await this.peelToCommit(rr.oid); if (!head) return errorPage(base, "no commit found"); - const found = this.lookupPath(head.commit.tree, path); + const found = await this.lookupPath(head.commit.tree, path); if (!found) return errorPage(base, `path not found: ${path.join("/")}`); const withPath = { ...base, pathBar: this.pathBar(repo, h, path) }; @@ -1153,14 +1251,14 @@ return htmlResponse(layout({ ...withPath, body })); } - private blobPage( + private async blobPage( repo: string, h: string | undefined, path: string[], entry: TreeEntry, base: Omit<LayoutOpts, "body"> - ): Response { - const obj = this.store.get(entry.oid); + ): Promise<Response> { + const obj = await this.store.get(entry.oid); if (!obj) return errorPage(base, "missing blob"); const r = `/${encodeURIComponent(repo)}`; const q = h ? `?h=${encodeURIComponent(h)}` : ""; @@ -1182,22 +1280,22 @@ return htmlResponse(layout({ ...base, body })); } - private blamePage(repo: string, h: string | undefined, path: string[]): Response { + private async blamePage(repo: string, h: string | undefined, path: string[]): Promise<Response> { const r = `/${encodeURIComponent(repo)}`; - const base = this.base(repo, "tree", h, `${r}/blame/${path.map(encodeURIComponent).join("/")}`); + const base = await this.base(repo, "tree", h, `${r}/blame/${path.map(encodeURIComponent).join("/")}`); const withPath = { ...base, pathBar: this.pathBar(repo, h, path) }; - const rr = this.resolveRef(h); + const rr = await this.resolveRef(h); if (!rr) return errorPage(withPath, "empty repository"); if (!path.length) return errorPage(withPath, "blame needs a file path"); // size-gate the tip blob before walking history: a large file would // otherwise load up to 200 full revisions into memory before blame() bails - const tip = this.peelToCommit(rr.oid); + const tip = await this.peelToCommit(rr.oid); if (!tip) return errorPage(withPath, "no commit found"); - const tipOid = this.pathOid(tip.commit, path); + const tipOid = await this.pathOid(tip.commit, path); const meta = tipOid ? this.store.typeAndSize(tipOid) : null; if (!meta || meta.type !== "blob") return errorPage(withPath, `no such file: ${path.join("/")}`); if (meta.size > MAX_BLAME_BYTES) return errorPage(withPath, "blame skipped: file too large"); - const history = this.pathHistory(rr.oid, path, 200, MAX_BLAME_HISTORY_BYTES); + const history = await this.pathHistory(rr.oid, path, 200, MAX_BLAME_HISTORY_BYTES); if (!history.length || !history[0].blob) return errorPage(withPath, `no such file: ${path.join("/")}`); if (isBinary(history[0].blob)) return errorPage(withPath, "cannot blame a binary file"); const result = blame(history); @@ -1223,37 +1321,37 @@ return htmlResponse(layout({ ...withPath, body })); } - private plainPage(h: string | undefined, path: string[]): Response { - const rr = this.resolveRef(h); + private async plainPage(h: string | undefined, path: string[]): Promise<Response> { + const rr = await this.resolveRef(h); if (!rr) return new Response("not found\n", { status: 404 }); - const head = this.peelToCommit(rr.oid); + const head = await this.peelToCommit(rr.oid); if (!head) return new Response("not found\n", { status: 404 }); - const found = this.lookupPath(head.commit.tree, path); + const found = await this.lookupPath(head.commit.tree, path); if (!found || found.kind !== "blob") return new Response("not found\n", { status: 404 }); - const obj = this.store.get(found.entry.oid); + const obj = await this.store.get(found.entry.oid); if (!obj) return new Response("not found\n", { status: 404 }); return rawBlobResponse(obj.data, path[path.length - 1] ?? ""); } - private blobByIdPage(id: string): Response { - const oid = this.resolveServable(id); + private async blobByIdPage(id: string): Promise<Response> { + const oid = await this.resolveServable(id); if (!oid) return new Response("not found\n", { status: 404 }); - const obj = this.store.get(oid); + const obj = await this.store.get(oid); if (!obj || obj.type !== "blob") return new Response("not a blob\n", { status: 404 }); return rawBlobResponse(obj.data, ""); } - private commitPage(repo: string, id: string | undefined, h: string | undefined): Response { + private async commitPage(repo: string, id: string | undefined, h: string | undefined): Promise<Response> { const r = `/${encodeURIComponent(repo)}`; - const base = this.base(repo, "commit", h, `${r}/commit/`); - const oid = this.resolveCommitId(id, h); + const base = await this.base(repo, "commit", h, `${r}/commit/`); + const oid = await this.resolveCommitId(id, h); if (!oid) return errorPage(base, "commit not found"); - const commit = this.loadCommit(oid); + const commit = await this.loadCommit(oid); if (!commit) return errorPage(base, `commit not found: ${oid}`); - const parent = commit.parents[0] ? this.loadCommit(commit.parents[0]) : null; - const { files, truncated } = this.computeDiff(parent?.tree ?? null, commit.tree); - const deco = this.decorations(repo); + const parent = commit.parents[0] ? await this.loadCommit(commit.parents[0]) : null; + const { files, truncated } = await this.computeDiff(parent?.tree ?? null, commit.tree); + const deco = await this.decorations(repo); const body = ` <table class='commit-info'> @@ -1270,30 +1368,30 @@ return htmlResponse(layout({ ...base, body })); } - private resolveCommitId(id: string | undefined, h: string | undefined): string | null { + private async resolveCommitId(id: string | undefined, h: string | undefined): Promise<string | null> { if (id) { - const full = this.resolveServable(id); - return full ? this.peelToCommit(full)?.oid ?? null : null; + const full = await this.resolveServable(id); + return full ? (await this.peelToCommit(full))?.oid ?? null : null; } - const rr = this.resolveRef(h); - return rr ? this.peelToCommit(rr.oid)?.oid ?? null : null; + const rr = await this.resolveRef(h); + return rr ? (await this.peelToCommit(rr.oid))?.oid ?? null : null; } /** diff/rawdiff: changes id2..id (default id2 = first parent of id). */ - private diffPage(repo: string, id: string | undefined, id2: string | undefined, h: string | undefined, raw: boolean): Response { - const base = this.base(repo, "diff", h, `/${encodeURIComponent(repo)}/diff/`); - const newOid = this.resolveCommitId(id, h); + private async diffPage(repo: string, id: string | undefined, id2: string | undefined, h: string | undefined, raw: boolean): Promise<Response> { + const base = await this.base(repo, "diff", h, `/${encodeURIComponent(repo)}/diff/`); + const newOid = await this.resolveCommitId(id, h); if (!newOid) return raw ? new Response("not found\n", { status: 404 }) : errorPage(base, "commit not found"); - const commit = this.loadCommit(newOid)!; + const commit = (await this.loadCommit(newOid))!; let oldOid: string | null = null; if (id2) { - oldOid = this.resolveCommitId(id2, undefined); + oldOid = await this.resolveCommitId(id2, undefined); if (!oldOid) return raw ? new Response("bad id2\n", { status: 404 }) : errorPage(base, `bad id2: ${id2}`); } else { oldOid = commit.parents[0] ?? null; } - const oldCommit = oldOid ? this.loadCommit(oldOid) : null; - const { files, truncated } = this.computeDiff(oldCommit?.tree ?? null, commit.tree); + const oldCommit = oldOid ? await this.loadCommit(oldOid) : null; + const { files, truncated } = await this.computeDiff(oldCommit?.tree ?? null, commit.tree); if (raw) { return new Response(this.renderRawDiff(files), { headers: { "content-type": "text/plain; charset=utf-8" } }); } @@ -1304,12 +1402,12 @@ } /** git-format-patch style output, applies with `git am`. */ - private patchPage(repo: string, id: string | undefined, h: string | undefined): Response { - const oid = this.resolveCommitId(id, h); + private async patchPage(repo: string, id: string | undefined, h: string | undefined): Promise<Response> { + const oid = await this.resolveCommitId(id, h); if (!oid) return new Response("not found\n", { status: 404 }); - const commit = this.loadCommit(oid)!; - const parent = commit.parents[0] ? this.loadCommit(commit.parents[0]) : null; - const { files } = this.computeDiff(parent?.tree ?? null, commit.tree); + const commit = (await this.loadCommit(oid))!; + const parent = commit.parents[0] ? await this.loadCommit(commit.parents[0]) : null; + const { files } = await this.computeDiff(parent?.tree ?? null, commit.tree); const bodyText = commit.message.split("\n").slice(1).join("\n").trim(); let statLines = ""; let totalAdd = 0, totalDel = 0; @@ -1334,13 +1432,13 @@ return new Response(patch, { headers: { "content-type": "text/plain; charset=utf-8" } }); } - private tagPage(repo: string, name: string | undefined): Response { + private async tagPage(repo: string, name: string | undefined): Promise<Response> { const r = `/${encodeURIComponent(repo)}`; - const base = this.base(repo, "refs", undefined, `${r}/refs/`); + const base = await this.base(repo, "refs", undefined, `${r}/refs/`); if (!name) return errorPage(base, "no tag given"); - const target = this.store.getRef(`refs/tags/${name}`) ?? this.resolveServable(name); + const target = this.store.getRef(`refs/tags/${name}`) ?? (await this.resolveServable(name)); if (!target) return errorPage(base, `tag not found: ${name}`); - const obj = this.store.get(target); + const obj = await this.store.get(target); if (!obj) return errorPage(base, `missing object`); let body: string; if (obj.type === "tag") { @@ -1363,7 +1461,7 @@ return htmlResponse(layout({ ...base, body })); } - private snapshotPage(repo: string, filename: string): Response { + private async snapshotPage(repo: string, filename: string): Promise<Response> { let format: "tar.gz" | "zip"; let stem: string; if (filename.endsWith(".tar.gz")) { @@ -1384,9 +1482,9 @@ for (const c of [...candidates]) candidates.push(`v${c}`); let commit: { oid: string; commit: Commit } | null = null; for (const cand of candidates) { - const rr = this.resolveRef(cand); + const rr = await this.resolveRef(cand); if (rr) { - commit = this.peelToCommit(rr.oid); + commit = await this.peelToCommit(rr.oid); if (commit) break; } } @@ -1393,11 +1491,11 @@ if (!commit) return new Response(`no ref matches snapshot name: ${stem}\n`, { status: 404 }); const flat = new Map<string, { oid: string; mode: string }>(); - this.flattenTree(commit.commit.tree, "", flat); + await this.flattenTree(commit.commit.tree, "", flat); const files: SnapshotFile[] = []; let total = 0; for (const [path, info] of flat) { - const obj = this.store.get(info.oid); + const obj = await this.store.get(info.oid); if (!obj) continue; total += obj.data.length; if (total > MAX_SNAPSHOT_BYTES) { @@ -1422,10 +1520,10 @@ }); } - private atomPage(repo: string, host: string, proto: string, h: string | undefined): Response { - const rr = this.resolveRef(h); + private async atomPage(repo: string, host: string, proto: string, h: string | undefined): Promise<Response> { + const rr = await this.resolveRef(h); if (!rr) return new Response("empty repository\n", { status: 404 }); - const { entries } = this.walkLog(rr.oid, 0, 20); + const { entries } = await this.walkLog(rr.oid, 0, 20); const abs = `${proto}://${host}/${encodeURIComponent(repo)}`; const iso = (t: number) => new Date(t * 1000).toISOString().replace(/\.\d+Z$/, "Z"); const updated = entries[0] ? iso(entries[0].commit.committer.time) : iso(0); @@ -1460,13 +1558,13 @@ * History under an immutable commit oid never changes, so a cache hit * needs no version check — only the pair itself as key. */ - private computeStats( + private async computeStats( tipOid: string, period: string - ): { periodKeys: string[]; authors: { name: string; counts: Record<string, number> }[]; totals: Record<string, number>; count: number } { + ): Promise<{ periodKeys: string[]; authors: { name: string; counts: Record<string, number> }[]; totals: Record<string, number>; count: number }> { const cached = this.store.getStatsCache(tipOid, period); if (cached) return JSON.parse(cached); - const { entries } = this.walkLog(tipOid, 0, MAX_STATS_SCAN); + const { entries } = await this.walkLog(tipOid, 0, MAX_STATS_SCAN); const keyOf = (t: number): string => { const d = new Date(t * 1000); if (period === "y") return String(d.getUTCFullYear()); @@ -1504,12 +1602,12 @@ return result; } - private statsPage(repo: string, h: string | undefined, period: string): Response { + private async statsPage(repo: string, h: string | undefined, period: string): Promise<Response> { const r = `/${encodeURIComponent(repo)}`; - const base = this.base(repo, "stats", h, `${r}/stats/`); - const rr = this.resolveRef(h); + const base = await this.base(repo, "stats", h, `${r}/stats/`); + const rr = await this.resolveRef(h); if (!rr) return errorPage(base, "empty repository"); - const agg = this.computeStats(rr.oid, period); + const agg = await this.computeStats(rr.oid, period); const cols = agg.periodKeys.slice(0, 8); const authors = agg.authors diff --git a/wrangler.jsonc b/wrangler.jsonc @@ -22,6 +22,12 @@ "migrations": [ { "tag": "v1", "new_sqlite_classes": ["RepoCell", "Registry"] } ], + // Optional: with PACK_CACHE bound, pushed pack bytes are stored in R2 (off the + // DO's SQLite) and full clones are served straight from R2 by the Worker. + // Remove this binding to fall back to the SQLite-only path (as on celld). + "r2_buckets": [ + { "binding": "PACK_CACHE", "bucket_name": "dgit-pack-cache" } + ], "vars": { "SITE_NAME": "git.littledivy.com", "SITE_DESC": "a fast webinterface for the git dscm, on Cloudflare", |