aboutsummaryrefslogtreecommitdiffstats
authorDivy Srivastava <me@littledivy.com>2026-08-19 13:47:03 +0530
committerDivy Srivastava <me@littledivy.com>2026-08-19 13:47:03 +0530
commit4fffc3a9fd6d04293af3494faf44c85d523a4dc4 (patch)
tree99e9f1005f8ee27391abd718aea3c8a2c6b3d295
parent0c364ef2f62ad36c5e72598973f2f44a17eb926c (diff)
downloaddgit-4fffc3a9fd.tar.gz zip
v0.0.2
Diffstat
README.md+23-7
src/env.ts+7-1
src/git/multipart.ts (new)+106-0
src/git/packstore.ts+279-105
src/git/protocol.ts+220-61
src/git/store.ts+11-2
src/index.ts+79-1
src/repo.ts+255-157
wrangler.jsonc+6-0
9 files changed, 986 insertions(+), 334 deletions(-)
diff --git a/README.md b/README.md
@@ -23,8 +23,13 @@
A push streams into the repository's cell and is stored as the packfile
the client sent; an index maps each object id to its pack, offset, and
delta base, so the client's compression is preserved rather than
-re-derived. A clone walks the closure of the requested refs and copies
-the stored compressed bytes verbatim into the outgoing pack. Fetch
+re-derived. When an R2 bucket is bound the pack bytes are written to R2
+and only the index stays in the cell's SQLite, so pack storage is no
+longer capped by the per-cell database. A clone walks the closure of the
+requested refs and copies the stored compressed bytes verbatim into the
+outgoing pack; a full clone, once built, is cached in R2 and afterwards
+streamed straight from the Worker, so repeat clones never load the cell.
+Fetch
negotiation excludes the closure of the client's haves, cut correctly
at shallow boundaries, so an incremental fetch downloads only what is
missing. Shallow clones (`--depth`, deepening, `--unshallow`), thin
@@ -44,10 +49,16 @@
```sh
npm install
+npx wrangler r2 bucket create dgit-pack-cache # optional: R2 pack offload + clone cache
npx wrangler deploy
npx wrangler secret put GIT_TOKEN # the push password
```
+The `PACK_CACHE` R2 binding in `wrangler.jsonc` is optional: with it,
+pushed pack bytes live in R2 (off the cell's SQLite) and full clones are
+served straight from R2 by the Worker without loading the cell. Remove
+the binding and everything falls back to the SQLite-only path.
+
Then push anything:
```sh
@@ -57,10 +68,12 @@
A Workers request is bounded at 128MB of memory and five minutes of
CPU, so a very large history lands as a series of smaller pushes rather
-than one; day-to-day pushes, clones, and fetches fit comfortably. A
-full-history clone of a repository with millions of objects can exceed
-the CPU bound — shallow and incremental fetches of the same repository
-are fine.
+than one; day-to-day pushes, clones, and fetches fit comfortably.
+Building a full-history clone of a repository with millions of objects
+can exceed the CPU bound the first time — once such a clone is cached in
+R2 it streams from the Worker without rebuilding, and shallow and
+incremental fetches of the same repository are fine regardless. The
+largest repositories belong on celld.
## Self-host on celld
@@ -91,7 +104,10 @@
Garbage collection also runs by itself, from a Durable Object alarm,
after a forced update or a ref deletion. `GIT_TOKENS` holds additional
-comma-separated tokens; `MAX_PUSH_MB` caps a single push.
+comma-separated tokens; `MAX_PUSH_MB` caps a single push. Setting
+`SHA1DC=1` screens every pushed object for a SHA-1 collision attack on
+ingest; by default objects are hashed with native SHA-1 — the same
+object ids, without the check.
## Contributions
diff --git a/src/env.ts b/src/env.ts
@@ -1,8 +1,11 @@
import type { Registry } from "./registry";
+import type { RepoCell } from "./repo";
export interface Env {
- REPO: DurableObjectNamespace;
+ REPO: DurableObjectNamespace<RepoCell>;
REGISTRY: DurableObjectNamespace<Registry>;
+ /** optional R2 bucket that offloads full-clone packs (absent on celld) */
+ PACK_CACHE?: R2Bucket;
/** shared secret required for git push (Basic auth password) */
GIT_TOKEN?: string;
/** optional comma-separated extra tokens (multiple users) */
@@ -9,6 +12,9 @@
GIT_TOKENS?: string;
/** max accepted push size in MB (default 512) */
MAX_PUSH_MB?: string;
+ /** "1" screens every pushed object with SHA-1DC collision detection; default
+ * hashes with native crypto.subtle (identical oid, far faster) */
+ SHA1DC?: string;
SITE_NAME?: string;
SITE_DESC?: string;
SITE_OWNER?: string;
diff --git a/src/git/multipart.ts b/src/git/multipart.ts
@@ -0,0 +1,106 @@
+import { concat } from "./util";
+
+/** In-progress R2 multipart upload for one pack object (clone cache or raw store). */
+export interface MultipartPackUpload {
+ uploadPart(data: Uint8Array): Promise<void>;
+ complete(): Promise<void>;
+ abort(): Promise<void>;
+}
+
+/** Uniform multipart part size. R2 requires every part but the last to be the
+ * same size and >= 5 MiB; 16 MiB parts cap a pack at 16 MiB * 10,000 = 160 GiB
+ * while holding at most one part in isolate memory at a time. */
+export const R2_PART_SIZE = 16 * 1024 * 1024;
+
+/**
+ * Buffers a streamed pack into fixed-size R2 multipart parts. Chunks are copied
+ * on push (callers hand out views into buffers that later reads evict). `drain`
+ * uploads every whole 16 MiB part and keeps the sub-part remainder, so isolate
+ * memory never holds more than ~one part; `finish` flushes the final (short)
+ * part and completes; any failure aborts so R2 is left with no partial object.
+ * Uniform part size (exactly R2_PART_SIZE, last excepted) is what satisfies R2's
+ * equal-size-parts rule.
+ */
+export class MultipartCapture {
+ private chunks: Uint8Array[] = [];
+ private len = 0;
+ private failed = false;
+ private completed = false;
+ constructor(private mp: MultipartPackUpload) {}
+
+ /** True once a part upload or complete/abort has failed (no R2 object exists). */
+ get aborted(): boolean {
+ return this.failed;
+ }
+
+ push(chunk: Uint8Array): void {
+ if (this.failed) return;
+ this.chunks.push(chunk.slice());
+ this.len += chunk.length;
+ }
+
+ async drain(): Promise<void> {
+ while (!this.failed && this.len >= R2_PART_SIZE) {
+ const merged = concat(this.chunks);
+ try {
+ await this.mp.uploadPart(merged.subarray(0, R2_PART_SIZE));
+ } catch {
+ await this.abort();
+ return;
+ }
+ const rest = merged.slice(R2_PART_SIZE);
+ this.chunks = rest.length ? [rest] : [];
+ this.len = rest.length;
+ }
+ }
+
+ async finish(): Promise<void> {
+ if (this.failed) return;
+ try {
+ if (this.len) await this.mp.uploadPart(concat(this.chunks));
+ await this.mp.complete();
+ this.completed = true;
+ } catch {
+ await this.abort();
+ }
+ this.chunks = [];
+ this.len = 0;
+ }
+
+ async abort(): Promise<void> {
+ if (this.failed || this.completed) return;
+ this.failed = true;
+ this.chunks = [];
+ this.len = 0;
+ try {
+ await this.mp.abort();
+ } catch {
+ // best effort: an un-aborted multipart auto-expires
+ }
+ }
+}
+
+/** Begin a raw R2 multipart upload under `key`. Returns null (caller falls back)
+ * if R2 refuses — celld deliberately makes createMultipartUpload throw. */
+export async function beginRawMultipart(
+ bucket: R2Bucket,
+ key: string
+): Promise<MultipartPackUpload | null> {
+ try {
+ const mp = await bucket.createMultipartUpload(key);
+ const parts: R2UploadedPart[] = [];
+ return {
+ uploadPart: async (data) => {
+ parts.push(await mp.uploadPart(parts.length + 1, data));
+ },
+ complete: async () => {
+ await mp.complete(parts);
+ },
+ abort: async () => {
+ await mp.abort();
+ },
+ };
+ } catch {
+ return null;
+ }
+}
diff --git a/src/git/packstore.ts b/src/git/packstore.ts
@@ -3,6 +3,7 @@
import { Sha1Dc, Sha1CollisionError } from "./sha1";
import { ObjType, NUM_TYPE, objectHeader } from "./objects";
import { applyDelta } from "./pack";
+import { MultipartCapture, beginRawMultipart } from "./multipart";
export const PACK_CHUNK = 1024 * 1024;
/** real Workers isolates have a hard 128MB total; self-hosted celld nodes run multi-GB heaps */
@@ -21,11 +22,27 @@
const RAW_CHUNK_CACHE = TIGHT_MEMORY ? 4 : 8;
/**
* Hard cap on delta base-chain length when resolving an object. git's default
- * pack depth is 50; 100 leaves generous headroom for legitimately deep packs
- * while a crafted deeper (or cyclic) chain is rejected instead of overflowing
- * the stack or looping forever.
+ * pack depth is 50; 50000 leaves enormous headroom for legitimately deep packs
+ * while a crafted unbounded chain is rejected instead of looping forever. The
+ * base walk is iterative (not recursive) and separately cycle-checked via a
+ * seen-set, so this bounds chain length rather than guarding stack depth.
*/
-const MAX_DELTA_DEPTH = 100;
+const MAX_DELTA_DEPTH = 50000;
+/**
+ * pako output-chunk size. The default (64 KiB) is large enough that V8 keeps
+ * every inflate's backing store in its array-buffer arena, so streaming a pack
+ * one Inflate per object commits arena proportional to the object count — a
+ * 53MB pack peaked near 250MB of buffers this way. 16 KiB chunks are collected
+ * promptly, holding inflate memory to a bounded working set regardless of pack
+ * size.
+ */
+const INFLATE_CHUNK = 16 * 1024;
+const inflateAll = (data: Uint8Array): Uint8Array => {
+ const inf = new pako.Inflate({ chunkSize: INFLATE_CHUNK });
+ inf.push(data, true);
+ if (inf.err) throw new Error(`inflate failed: ${inf.msg}`);
+ return inf.result as Uint8Array;
+};
export interface ObjRec {
type: ObjType;
@@ -75,12 +92,15 @@
}
}
-// ingest is strictly sequential, so scratch hashers serve every object
+// ingest is strictly sequential, so scratch hashers serve every object. The
+// SHA-1DC pair is used only when collision detection is enabled; the plain
+// streaming hasher covers the flag-off huge-object path.
const scratchSha = new Sha1Dc();
const streamSha = new Sha1Dc();
+const streamShaPlain = new Sha1();
/**
- * Finish an object hash, refusing anything carrying a SHA-1 collision-attack
+ * Finish a SHA-1DC hash, refusing anything carrying a SHA-1 collision-attack
* block. This is git's post-SHAttered rule: the id is still plain SHA-1, but
* an object built to collide never enters the database.
*/
@@ -91,64 +111,103 @@
}
/**
- * Every ingested object goes through the collision-detecting hash, so there
- * is no fast path here: crypto.subtle is ~10x quicker on large blobs but
- * cannot screen for near-collision blocks, and an attacker would only have to
- * pad past the threshold to skip the check.
+ * Hash a fully-materialized object. With collision detection off (default) the
+ * id comes from native crypto.subtle — byte-identical to plain SHA-1 but ~24x
+ * faster than the JS hasher; with it on, SHA-1DC screens every block. The
+ * object is already in memory here, so the one-shot header+data buffer adds no
+ * unbounded allocation.
*/
-async function hashObjectAsync(type: ObjType, data: Uint8Array): Promise<string> {
- return finishOid(scratchSha.reset().update(objectHeader(type, data.length)).update(data));
+async function hashObjectAsync(type: ObjType, data: Uint8Array, cd: boolean): Promise<string> {
+ if (cd) return finishOid(scratchSha.reset().update(objectHeader(type, data.length)).update(data));
+ const header = objectHeader(type, data.length);
+ const buf = new Uint8Array(header.length + data.length);
+ buf.set(header);
+ buf.set(data, header.length);
+ return toHex(new Uint8Array(await crypto.subtle.digest("SHA-1", buf)));
}
-/** Buffered sequential reader over a pack stored as chunk rows. */
+/**
+ * Buffered sequential reader over a pack stored as SQLite chunk rows or an R2
+ * object. Chunks are loaded through the owning PackStore's chunk cache (async
+ * because an R2-backed chunk is a range GET), so byte()/window() await; the SQL
+ * backend resolves without I/O.
+ */
class PackReader {
pos = 0;
private seq = -1;
private chunk: Uint8Array = new Uint8Array(0);
- constructor(private sql: SqlStorage, private packId: number, readonly size: number) {}
+ constructor(private load: (seq: number) => Promise<Uint8Array>, readonly size: number) {}
- private ensure(): void {
+ private async ensure(): Promise<void> {
const want = Math.floor(this.pos / PACK_CHUNK);
if (want !== this.seq) {
- const rows = this.sql
- .exec<{ data: ArrayBuffer }>(
- "SELECT data FROM pack_data WHERE pack_id = ? AND seq = ?",
- this.packId,
- want
- )
- .toArray();
- if (!rows.length) throw new Error(`pack ${this.packId}: missing chunk ${want}`);
- this.chunk = new Uint8Array(rows[0].data);
+ this.chunk = await this.load(want);
this.seq = want;
}
}
- byte(): number {
- this.ensure();
+ async byte(): Promise<number> {
+ await this.ensure();
return this.chunk[this.pos++ - this.seq * PACK_CHUNK];
}
/** Remaining bytes of the chunk containing pos (never empty while pos < size). */
- window(): Uint8Array {
- this.ensure();
+ async window(): Promise<Uint8Array> {
+ await this.ensure();
return this.chunk.subarray(this.pos - this.seq * PACK_CHUNK);
}
}
/**
- * Pack-native object database: received packfiles are stored verbatim in
- * chunk rows and indexed (oid -> pack/offset/base), preserving the client's
- * delta compression. This is what lets Linux-sized repos fit and stream.
+ * Pack-native object database: received packfiles are stored verbatim — in R2
+ * under `raw/<repo>/<packId>` when a bucket is bound (PACK_CACHE), else in
+ * SQLite chunk rows (the fallback / celld path) — and indexed (oid ->
+ * pack/offset/base), preserving the client's delta compression. Only the index
+ * ever lives in SQLite for an R2-backed pack, so pack data escapes the per-DO
+ * storage ceiling. This is what lets large repos fit and stream.
*/
export class PackStore {
- /** LRU of decoded `pack_data` rows, keyed "packId:seq". */
+ /** LRU of decoded pack chunks, keyed "packId:seq" (SQLite row or R2 range). */
private chunks = new Map<string, Uint8Array>();
+ /**
+ * R2 raw-pack backend for this repo, or null when no bucket is bound (celld /
+ * Workers without the binding). When set, newly ingested packs store their
+ * bytes in R2 under `raw/<repo>/<packId>` and keep only the index in SQLite;
+ * pre-existing SQLite packs keep working (backend is recorded per pack).
+ */
+ private r2: { bucket: R2Bucket; repo: string } | null = null;
+ /** Cached backend ('sql' | 'r2') per pack id, from pack_meta.store. */
+ private backend = new Map<number, "sql" | "r2">();
constructor(private sql: SqlStorage, private extern: ExternalResolver) {
this.init();
}
+ /** Bind (or clear) the R2 backend. Idempotent; the DO is per-repo so this is
+ * set on each request from the repo name. */
+ setR2(bucket: R2Bucket | undefined, repo: string): void {
+ this.r2 = bucket ? { bucket, repo } : null;
+ }
+
+ private rawKey(packId: number): string {
+ return `raw/${this.r2!.repo}/${packId}`;
+ }
+
+ /** Storage backend of a pack. Cached; falls back to a pack_meta lookup. The
+ * cache is primed at ingest start (before pack_meta exists) so phase B/C read
+ * the right source. */
+ private backendOf(packId: number): "sql" | "r2" {
+ const hit = this.backend.get(packId);
+ if (hit) return hit;
+ const rows = this.sql
+ .exec<{ store: string }>("SELECT store FROM pack_meta WHERE pack_id = ?", packId)
+ .toArray();
+ const b = rows[0]?.store === "r2" ? "r2" : "sql";
+ this.backend.set(packId, b);
+ return b;
+ }
+
private init(): void {
this.sql.exec(`
CREATE TABLE IF NOT EXISTS pack_meta (
@@ -155,7 +214,8 @@
pack_id INTEGER PRIMARY KEY,
size INTEGER NOT NULL,
count INTEGER NOT NULL,
- created INTEGER NOT NULL
+ created INTEGER NOT NULL,
+ store TEXT NOT NULL DEFAULT 'sql'
);
CREATE TABLE IF NOT EXISTS pack_data (
pack_id INTEGER NOT NULL,
@@ -186,6 +246,12 @@
PRIMARY KEY (pack_id, offset)
);
`);
+ // back-fill the store column on repos whose pack_meta predates the R2 backend:
+ // every such pack is SQLite-backed, which the 'sql' default records
+ const cols = this.sql.exec<{ name: string }>("PRAGMA table_info(pack_meta)").toArray();
+ if (!cols.some((c) => c.name === "store")) {
+ this.sql.exec("ALTER TABLE pack_meta ADD COLUMN store TEXT NOT NULL DEFAULT 'sql'");
+ }
}
wipe(): void {
@@ -193,6 +259,7 @@
this.sql.exec(`DROP TABLE IF EXISTS ${t}`);
}
this.chunks.clear(); // pack ids restart from 1: cached rows would be stale
+ this.backend.clear();
}
/** Drop all packs and start empty (small-repo gc migrates objects out first). */
@@ -201,6 +268,27 @@
this.init();
}
+ /** Pack ids of the currently R2-backed packs (for GC to delete their R2 bytes). */
+ r2PackIds(): number[] {
+ return this.sql
+ .exec<{ pack_id: number }>("SELECT pack_id FROM pack_meta WHERE store = 'r2'")
+ .toArray()
+ .map((r) => r.pack_id);
+ }
+
+ /** Delete the R2 raw objects for the given pack ids (no-op without a binding). */
+ async deleteR2Packs(packIds: number[]): Promise<void> {
+ if (!this.r2 || !packIds.length) return;
+ const keys = packIds.map((id) => this.rawKey(id));
+ for (let i = 0; i < keys.length; i += 100) {
+ try {
+ await this.r2.bucket.delete(keys.slice(i, i + 100));
+ } catch {
+ // best effort: an undeleted raw object is unreferenced garbage
+ }
+ }
+ }
+
countObjects(): number {
return this.sql.exec<{ n: number }>("SELECT COUNT(*) AS n FROM pack_objects").one().n;
}
@@ -265,12 +353,12 @@
* fetched one at a time: a wide range in a single SELECT can exceed the
* runtime's result-set size cap (celld materializes blob rows as JSON).
*/
- readRaw(packId: number, off: number, len: number): Uint8Array {
+ async readRaw(packId: number, off: number, len: number): Promise<Uint8Array> {
const first = Math.floor(off / PACK_CHUNK);
const last = Math.floor((off + len - 1) / PACK_CHUNK);
const out = new Uint8Array(len);
for (let seq = first; seq <= last; seq++) {
- const chunk = this.chunk(packId, seq);
+ const chunk = await this.chunk(packId, seq);
const chunkStart = seq * PACK_CHUNK;
const from = Math.max(off, chunkStart);
const to = Math.min(off + len, chunkStart + chunk.length);
@@ -280,12 +368,15 @@
}
/**
- * One decoded `pack_data` row, through the LRU. Rows are immutable once
- * written — a pack chunk is inserted exactly once and only ever removed
- * wholesale (wipe/reset, or the orphan sweep at the top of ingest, both of
- * which clear the cache), so a hit can never be stale.
+ * One decoded pack chunk, through the LRU. For a SQLite-backed pack this is a
+ * `pack_data` row; for an R2-backed pack it is an aligned ~1 MiB range GET
+ * against `raw/<repo>/<packId>`, so sorted sequential reads (a clone) issue
+ * ~one GET per chunk instead of one per object. Chunks are immutable once
+ * written — a pack is inserted once and only ever removed wholesale (wipe/
+ * reset, or the orphan sweep at the top of ingest, both of which clear the
+ * cache), so a hit can never be stale.
*/
- private chunk(packId: number, seq: number): Uint8Array {
+ private async chunk(packId: number, seq: number): Promise<Uint8Array> {
const key = `${packId}:${seq}`;
const hit = this.chunks.get(key);
if (hit) {
@@ -293,15 +384,25 @@
this.chunks.set(key, hit);
return hit;
}
- const rows = this.sql
- .exec<{ data: ArrayBuffer }>(
- "SELECT data FROM pack_data WHERE pack_id = ? AND seq = ?",
- packId,
- seq
- )
- .toArray();
- if (!rows.length) throw new Error(`pack ${packId}: missing chunk ${seq}`);
- const chunk = new Uint8Array(rows[0].data);
+ let chunk: Uint8Array;
+ if (this.backendOf(packId) === "r2") {
+ if (!this.r2) throw new Error(`pack ${packId}: R2 backend not bound`);
+ const obj = await this.r2.bucket.get(this.rawKey(packId), {
+ range: { offset: seq * PACK_CHUNK, length: PACK_CHUNK },
+ });
+ if (!obj) throw new Error(`pack ${packId}: missing R2 object ${this.rawKey(packId)}`);
+ chunk = new Uint8Array(await obj.arrayBuffer());
+ } else {
+ const rows = this.sql
+ .exec<{ data: ArrayBuffer }>(
+ "SELECT data FROM pack_data WHERE pack_id = ? AND seq = ?",
+ packId,
+ seq
+ )
+ .toArray();
+ if (!rows.length) throw new Error(`pack ${packId}: missing chunk ${seq}`);
+ chunk = new Uint8Array(rows[0].data);
+ }
this.chunks.set(key, chunk);
while (this.chunks.size > RAW_CHUNK_CACHE) {
this.chunks.delete(this.chunks.keys().next().value as string);
@@ -317,7 +418,7 @@
* then applied from the base upward, holding at most two inflated buffers at
* a time. This caps both stack depth and peak memory regardless of input.
*/
- getObject(oid: string, cache: ObjCache): ObjRec | null {
+ async getObject(oid: string, cache: ObjCache): Promise<ObjRec | null> {
const cached = cache.get(oid);
if (cached) return cached;
const first = this.lookup(oid);
@@ -334,7 +435,7 @@
if (seen.has(cur.oid)) throw new Error(`cyclic delta chain at ${cur.oid}`);
seen.add(cur.oid);
if (!cur.baseOid) {
- base = { type: cur.type, data: pako.inflate(this.readRaw(cur.packId, cur.dataOff, cur.dataLen)) };
+ base = { type: cur.type, data: inflateAll(await this.readRaw(cur.packId, cur.dataOff, cur.dataLen)) };
break;
}
if (chain.length >= MAX_DELTA_DEPTH) throw new Error(`delta chain exceeds depth ${MAX_DELTA_DEPTH} at ${oid}`);
@@ -355,7 +456,7 @@
let obj = base;
for (let i = chain.length - 1; i >= 0; i--) {
const d = chain[i];
- const delta = pako.inflate(this.readRaw(d.packId, d.dataOff, d.dataLen));
+ const delta = inflateAll(await this.readRaw(d.packId, d.dataOff, d.dataLen));
obj = { type: base.type, data: applyDelta(obj.data, delta) };
cache.put(d.oid, obj);
}
@@ -364,8 +465,9 @@
}
/**
- * Ingest a packfile arriving as a byte stream: store chunks verbatim
- * (verifying the SHA-1 trailer on the fly), then index sequentially.
+ * Ingest a packfile arriving as a byte stream: store the bytes verbatim
+ * (to R2 when a bucket is bound, else SQLite chunk rows) while verifying the
+ * SHA-1 trailer on the fly, then index sequentially.
* Deltas resolve eagerly against the LRU cache — pack ordering keeps
* bases hot — so the straggler pass afterwards is nearly empty. Memory
* stays bounded regardless of pack size.
@@ -380,13 +482,24 @@
/** called periodically so the runtime can flush its write buffer —
* workerd holds a request's dirty pages in the 128MB isolate heap */
flush?: () => Promise<void>;
+ /** screen every block with SHA-1DC (git-parity collision detection);
+ * default hashes with native crypto.subtle, which yields the identical
+ * oid without the collision check */
+ collisionDetect?: boolean;
}
): Promise<{ packId: number; count: number }> {
- // a failed or interrupted ingest leaves rows without a pack_meta entry;
- // reclaim that space before starting (pack_meta is only written on success)
+ const cd = opts.collisionDetect ?? false;
+ // a failed or interrupted ingest leaves rows (or an R2 raw object) without a
+ // pack_meta entry; reclaim that space before starting (pack_meta is only
+ // written on success). Union across every per-pack table so an R2 ingest that
+ // died mid-index — whose bytes are in R2, not pack_data — is caught too.
const orphans = this.sql
.exec<{ pack_id: number }>(
- "SELECT DISTINCT d.pack_id AS pack_id FROM pack_data d LEFT JOIN pack_meta m ON m.pack_id = d.pack_id WHERE m.pack_id IS NULL"
+ `SELECT pack_id FROM (
+ SELECT DISTINCT pack_id FROM pack_data
+ UNION SELECT DISTINCT pack_id FROM pack_objects
+ UNION SELECT DISTINCT pack_id FROM pack_pending
+ ) WHERE pack_id NOT IN (SELECT pack_id FROM pack_meta)`
)
.toArray();
for (const o of orphans) {
@@ -393,6 +506,14 @@
for (const t of ["pack_data", "pack_objects", "pack_pending"]) {
this.sql.exec(`DELETE FROM ${t} WHERE pack_id = ?`, o.pack_id);
}
+ // an interrupted R2 ingest may have left a completed raw object; delete it
+ if (this.r2) {
+ try {
+ await this.r2.bucket.delete(this.rawKey(o.pack_id));
+ } catch {
+ // best effort: the packId is reused next and its raw key overwritten
+ }
+ }
}
// the reclaimed pack id is about to be handed out again: drop anything the
// chunk LRU still holds for it (and for anything else — it is only a cache)
@@ -405,7 +526,20 @@
console.log(`[ingest pack ${packId} +${Math.round((Date.now() - started) / 1000)}s] ${msg.trim()}`);
};
- // phase A: stream bytes into chunk rows, hashing all but the trailing 20.
+ // storage backend: with an R2 binding the pack bytes stream to
+ // raw/<repo>/<packId> via multipart and only the index stays in SQLite; else
+ // they go to pack_data as before. Prime the backend cache now — before
+ // pack_meta exists — so phase B/C readRaw hits the right source. A refused
+ // multipart begin (celld makes createMultipartUpload throw) degrades to SQLite.
+ let capture: MultipartCapture | null = null;
+ if (this.r2) {
+ const mp = await beginRawMultipart(this.r2.bucket, this.rawKey(packId));
+ if (mp) capture = new MultipartCapture(mp);
+ }
+ const r2Backed = capture !== null;
+ this.backend.set(packId, r2Backed ? "r2" : "sql");
+
+ // phase A: stream bytes into R2 (or chunk rows), hashing all but the trailing 20.
// Strictly linear: a cursor fills a fixed 1MB buffer — the source may
// arrive as one multi-GB chunk (celld buffers request bodies), so any
// re-concatenation of the remainder would go quadratic.
@@ -430,6 +564,10 @@
} else {
tail = joined.slice();
}
+ if (capture) {
+ capture.push(data); // copied into a 16 MiB multipart part buffer
+ return;
+ }
let off = 0;
while (off < data.length) {
const take = Math.min(PACK_CHUNK - chunkLen, data.length - off);
@@ -453,6 +591,10 @@
const feedSliced = async (data: Uint8Array) => {
for (let off = 0; off < data.length; off += FEED_SLICE) {
feed(data.subarray(off, off + FEED_SLICE));
+ if (capture) {
+ await capture.drain(); // upload whole 16 MiB parts, keep only the remainder
+ if (capture.aborted) throw new Error("pack upload to R2 failed");
+ }
if (data.length > FEED_SLICE) await new Promise((res) => setTimeout(res, 0));
await opts.flush?.();
}
@@ -465,7 +607,7 @@
if (value?.length) await feedSliced(value);
}
}
- if (chunkLen > 0) {
+ if (!capture && chunkLen > 0) {
this.sql.exec(
"INSERT INTO pack_data (pack_id, seq, data) VALUES (?, ?, ?)",
packId,
@@ -473,32 +615,59 @@
chunkBuf.slice(0, chunkLen).buffer
);
}
- if (total < 32) throw new Error("pack too small");
- if (toHex(tail) !== toHex(sha.digest())) throw new Error("pack checksum mismatch");
+ // validate the trailer before completing the R2 upload, so a corrupt push
+ // never leaves a finished raw object (abort leaves R2 with nothing).
+ if (total < 32) {
+ if (capture) await capture.abort();
+ throw new Error("pack too small");
+ }
+ if (toHex(tail) !== toHex(sha.digest())) {
+ if (capture) await capture.abort();
+ throw new Error("pack checksum mismatch");
+ }
+ if (capture) {
+ // complete the multipart BEFORE phase B/C: both readRaw the finished raw
+ // object (phase B for eager delta bases, phase C for deferred deltas).
+ await capture.finish();
+ if (capture.aborted) throw new Error("pack upload to R2 failed");
+ }
say(`Received pack: ${Math.round(total / 1048576)}MB\n`);
// phase B: sequential scan with eager delta resolution
- const r = new PackReader(this.sql, packId, total);
+ const r = new PackReader((s) => this.chunk(packId, s), total);
const magic = new Uint8Array(4);
- for (let i = 0; i < 4; i++) magic[i] = r.byte();
+ for (let i = 0; i < 4; i++) magic[i] = await r.byte();
if (td.decode(magic) !== "PACK") throw new Error("bad pack signature");
let version = 0, count = 0;
- for (let i = 0; i < 4; i++) version = (version << 8) | r.byte();
- for (let i = 0; i < 4; i++) count = (count << 8) | r.byte();
+ for (let i = 0; i < 4; i++) version = (version << 8) | (await r.byte());
+ for (let i = 0; i < 4; i++) count = (count << 8) | (await r.byte());
if (version !== 2 && version !== 3) throw new Error(`unsupported pack version ${version}`);
- // single-row statements: celld's SQLite caps bound variables far lower
- // than workerd, and in-process inserts are cheap even at Linux scale
- const insertObject = (row: (string | number | null)[]) => {
- // OR IGNORE, never OR REPLACE: a duplicate object must keep its FIRST
- // location. Re-pointing it at the incoming pack means the orphan sweep
- // erases it from the index if this ingest fails — objects that were
- // safely stored become unresolvable ("missing base") forever after.
+ // batched index inserts: one exec per object made pack_objects a top SQL
+ // cost, and deferring rows is safe because delta bases resolve from the
+ // in-memory offset window and object cache — the flush points below cover
+ // the rare pack_objects SELECT fallbacks (oidByOffset miss, getObject).
+ // OR IGNORE keeps a duplicate oid's FIRST location — including within one
+ // batch — so a failed ingest's orphan sweep can't strand a stored object.
+ // workerd caps a DO SQL statement at ~100 bound variables (celld's stock
+ // SQLite allows more); 10 rows x 9 cols = 90 stays under that on both.
+ const OBJ_BATCH = 10;
+ let objBatch: (string | number | null)[] = [];
+ let objBatchRows = 0;
+ const flushObjects = () => {
+ if (!objBatchRows) return;
+ const values = Array(objBatchRows).fill("(?,?,?,?,?,?,?,?,?)").join(",");
this.sql.exec(
- "INSERT OR IGNORE INTO pack_objects (oid, pack_id, offset, data_off, data_len, type, size, entry_size, base_oid) VALUES (?,?,?,?,?,?,?,?,?)",
- ...row
+ `INSERT OR IGNORE INTO pack_objects (oid, pack_id, offset, data_off, data_len, type, size, entry_size, base_oid) VALUES ${values}`,
+ ...objBatch
);
+ objBatch = [];
+ objBatchRows = 0;
};
+ const insertObject = (row: (string | number | null)[]) => {
+ for (const v of row) objBatch.push(v);
+ if (++objBatchRows >= OBJ_BATCH) flushObjects();
+ };
const insertPending = (row: (string | number | null)[]) => {
this.sql.exec(
"INSERT INTO pack_pending (pack_id, offset, data_off, data_len, entry_size, base_oid, base_offset) VALUES (?,?,?,?,?,?,?)",
@@ -519,6 +688,7 @@
const oidByOffset = (off: number): string | null => {
const hit = winCur.get(off) ?? winPrev.get(off);
if (hit) return hit;
+ flushObjects(); // a windowed miss falls back to SELECT: deferred rows must be visible
const rows = this.sql
.exec<{ oid: string }>("SELECT oid FROM pack_objects WHERE pack_id = ? AND offset = ?", packId, off)
.toArray();
@@ -528,12 +698,12 @@
let pendingCount = 0;
for (let i = 0; i < count; i++) {
const offset = r.pos;
- let byte = r.byte();
+ let byte = await r.byte();
const type = (byte >> 4) & 7;
let entrySize = byte & 15;
let shift = 4;
while (byte & 0x80) {
- byte = r.byte();
+ byte = await r.byte();
entrySize += (byte & 0x7f) * 2 ** shift;
shift += 7;
}
@@ -540,16 +710,16 @@
let baseOid: string | null = null;
let baseOffset: number | null = null;
if (type === 6) {
- byte = r.byte();
+ byte = await r.byte();
let off = byte & 0x7f;
while (byte & 0x80) {
- byte = r.byte();
+ byte = await r.byte();
off = (off + 1) * 128 + (byte & 0x7f);
}
baseOffset = offset - off;
} else if (type === 7) {
const b = new Uint8Array(20);
- for (let j = 0; j < 20; j++) b[j] = r.byte();
+ for (let j = 0; j < 20; j++) b[j] = await r.byte();
baseOid = toHex(b);
} else if (!NUM_TYPE[type]) {
throw new Error(`bad object type ${type} at ${offset}`);
@@ -559,19 +729,24 @@
const isFull = type >= 1 && type <= 4;
const objType = isFull ? NUM_TYPE[type] : null;
const buffered = entrySize <= MAX_BUFFERED_ENTRY;
- const pieces: Uint8Array[] = [];
- if (isFull && !buffered) streamSha.reset().update(objectHeader(objType!, entrySize));
+ // buffered entries inflate straight into one exact-size buffer (no piece
+ // array + concat, which would hold the object twice at once)
+ const content = buffered ? new Uint8Array(entrySize) : null;
+ if (isFull && !buffered) {
+ if (cd) streamSha.reset().update(objectHeader(objType!, entrySize));
+ else streamShaPlain.reset().update(objectHeader(objType!, entrySize));
+ }
let inflated = 0;
- const inf = new pako.Inflate();
+ const inf = new pako.Inflate({ chunkSize: INFLATE_CHUNK });
(inf as unknown as { onData: (c: Uint8Array) => void }).onData = (c: Uint8Array) => {
+ if (content) content.set(c, inflated);
+ else if (isFull) (cd ? streamSha : streamShaPlain).update(c);
inflated += c.length;
- if (buffered) pieces.push(c);
- else if (isFull) streamSha.update(c);
};
const anyInf = inf as unknown as { err: number; msg: string; ended: boolean; strm: { avail_in: number } };
while (!anyInf.ended) {
if (r.pos >= total) throw new Error(`truncated pack entry at ${offset}`);
- const win = r.window();
+ const win = await r.window();
inf.push(win, false);
if (anyInf.err) throw new Error(`inflate failed at ${dataOff}: ${anyInf.msg}`);
r.pos += anyInf.ended ? win.length - anyInf.strm.avail_in : win.length;
@@ -580,14 +755,11 @@
const dataLen = r.pos - dataOff;
if (isFull) {
- let content: Uint8Array | null = null;
- let oid: string;
- if (buffered) {
- content = pieces.length === 1 ? pieces[0] : concat(pieces);
- oid = await hashObjectAsync(objType!, content);
- } else {
- oid = finishOid(streamSha);
- }
+ const oid = buffered
+ ? await hashObjectAsync(objType!, content!, cd)
+ : cd
+ ? finishOid(streamSha)
+ : toHex(streamShaPlain.digest());
insertObject([oid, packId, offset, dataOff, dataLen, objType!, entrySize, entrySize, null]);
winPut(offset, oid);
if (content && content.length <= CACHE_ENTRY_LIMIT) {
@@ -599,8 +771,9 @@
if (buffered && resolvedBase) {
let base: ObjRec | null = opts.cache.get(resolvedBase) ?? null;
if (!base) {
+ flushObjects(); // getObject SELECTs pack_objects: deferred rows must be visible
try {
- base = this.getObject(resolvedBase, opts.cache);
+ base = await this.getObject(resolvedBase, opts.cache);
} catch {
base = null;
}
@@ -607,13 +780,12 @@
}
if (!base) base = this.extern(resolvedBase);
if (base) {
- const delta = pieces.length === 1 ? pieces[0] : concat(pieces);
- const content = applyDelta(base.data, delta);
- const oid = await hashObjectAsync(base.type, content);
- insertObject([oid, packId, offset, dataOff, dataLen, base.type, content.length, entrySize, resolvedBase]);
+ const data = applyDelta(base.data, content!);
+ const oid = await hashObjectAsync(base.type, data, cd);
+ insertObject([oid, packId, offset, dataOff, dataLen, base.type, data.length, entrySize, resolvedBase]);
winPut(offset, oid);
- if (content.length <= CACHE_ENTRY_LIMIT) {
- opts.cache.put(oid, { type: base.type, data: content });
+ if (data.length <= CACHE_ENTRY_LIMIT) {
+ opts.cache.put(oid, { type: base.type, data });
}
resolved = true;
}
@@ -666,15 +838,16 @@
}
if (!baseOid) continue;
let base: ObjRec | null = null;
+ flushObjects(); // getObject SELECTs pack_objects: deferred rows must be visible
try {
- base = this.getObject(baseOid, opts.cache) ?? this.extern(baseOid);
+ base = (await this.getObject(baseOid, opts.cache)) ?? this.extern(baseOid);
} catch {
base = null;
}
if (!base) continue; // thin base not available (yet)
- const delta = pako.inflate(this.readRaw(packId, row.data_off, row.data_len));
+ const delta = inflateAll(await this.readRaw(packId, row.data_off, row.data_len));
const data = applyDelta(base.data, delta);
- const oid = await hashObjectAsync(base.type, data);
+ const oid = await hashObjectAsync(base.type, data, cd);
insertObject([oid, packId, row.offset, row.data_off, row.data_len, base.type, data.length, row.entry_size, baseOid]);
this.sql.exec("DELETE FROM pack_pending WHERE pack_id = ? AND offset = ?", packId, row.offset);
winPut(row.offset, oid);
@@ -693,10 +866,11 @@
}
}
if (resolvedTotal) say(`Resolved ${resolvedTotal} deferred delta(s)\n`);
+ flushObjects(); // commit the final partial batch before pack_meta marks the pack complete
this.sql.exec(
- "INSERT INTO pack_meta (pack_id, size, count, created) VALUES (?, ?, ?, ?)",
- packId, total, count, Date.now()
+ "INSERT INTO pack_meta (pack_id, size, count, created, store) VALUES (?, ?, ?, ?, ?)",
+ packId, total, count, Date.now(), r2Backed ? "r2" : "sql"
);
return { packId, count };
}
diff --git a/src/git/protocol.ts b/src/git/protocol.ts
@@ -3,6 +3,7 @@
import { GitStore } from "./store";
import { PackWriter } from "./pack";
import { OidSet } from "./oidset";
+import { MultipartCapture, MultipartPackUpload } from "./multipart";
import { parseCommit, parseTag, parseTree, isGitlinkMode, isTreeMode, TYPE_NUM, NUM_TYPE } from "./objects";
const AGENT = "agent=dgit/0.3";
@@ -20,6 +21,20 @@
export type Service = "git-upload-pack" | "git-receive-pack";
+/**
+ * Optional R2-backed full-clone offload, write side only: on a full-clone MISS
+ * the DO tees the generated pack into a multipart upload so it never buffers the
+ * whole pack in isolate memory (no size cap). Absent on celld and on Workers
+ * without the binding, in which case every clone is served from the DO exactly
+ * as before. HITs are served by the Worker streaming R2 directly, not the DO.
+ */
+export interface PackCache {
+ repo: string;
+ /** Begin a multipart pack build under `key`, stamping the object count as
+ * customMetadata; null when R2 is unavailable (the clone still streams). */
+ beginMultipart(key: string, objects: number): Promise<MultipartPackUpload | null>;
+}
+
/** GET /info/refs?service=... — smart ref advertisement (protocol v0). */
export function advertisement(store: GitStore, service: Service): Uint8Array {
const caps =
@@ -64,7 +79,7 @@
* who learned its sha. Membership in a set of ref tips, so a normal fetch
* (which only ever wants advertised tips) pays a few hash lookups, not a walk.
*/
-export function advertisedOids(store: GitStore): Set<string> {
+export async function advertisedOids(store: GitStore): Promise<Set<string>> {
const oids = new Set<string>();
const head = store.resolveHead();
if (head) oids.add(head);
@@ -73,7 +88,7 @@
// peel annotated tags; lightweight tags already point at the commit
if (!r.name.startsWith("refs/tags/")) continue;
if (store.typeAndSize(r.target)?.type !== "tag") continue;
- const peeled = peelToCommitOid(store, r.target);
+ const peeled = await peelToCommitOid(store, r.target);
if (peeled) oids.add(peeled);
}
return oids;
@@ -85,7 +100,7 @@
* cache describe precisely what the client asked for; a subset of the tips would
* pull unrelated branches' objects in and leave the client with danglers.
*/
-function wantsAreAllTips(store: GitStore, wants: string[]): boolean {
+export function wantsAreAllTips(store: GitStore, wants: string[]): boolean {
const tips = new Set<string>();
for (const r of store.refs()) tips.add(r.target);
if (tips.size === 0) return false;
@@ -107,6 +122,57 @@
return frames;
}
+/**
+ * Serve a full clone by streaming the R2-cached raw pack straight from the
+ * Worker: the NAK preamble a full-clone negotiation always produces, the
+ * optional band-2 progress line, then the pack re-chunked into band-1 side-band
+ * frames as R2 hands it over, then flush. Framing is identical to the DO path
+ * and the pack bytes are the stored bytes, so the client receives a
+ * byte-for-byte-equal pack. Memory is one R2 read chunk at a time — never
+ * proportional to pack size — and the DO is never contacted for bytes.
+ */
+export function streamingCloneResponse(
+ body: ReadableStream<Uint8Array>,
+ objects: number,
+ noProgress: boolean
+): Response {
+ const headers = {
+ "content-type": "application/x-git-upload-pack-result",
+ "cache-control": "no-cache",
+ "x-pack-cache": "hit",
+ };
+ const reader = body.getReader();
+ let started = false;
+ let done = false;
+ const stream = new ReadableStream<Uint8Array>({
+ pull: async (ctrl) => {
+ try {
+ if (!started) {
+ started = true;
+ ctrl.enqueue(pkt("NAK\n"));
+ if (!noProgress) {
+ for (const f of sidebandFrames(2, te.encode(`Enumerating objects: ${objects}, done.\n`))) ctrl.enqueue(f);
+ }
+ }
+ const { done: rdone, value } = await reader.read();
+ if (rdone) {
+ if (!done) {
+ done = true;
+ ctrl.enqueue(FLUSH);
+ ctrl.close();
+ }
+ return;
+ }
+ for (const f of sidebandFrames(1, value)) ctrl.enqueue(f);
+ } catch (err) {
+ ctrl.error(err);
+ }
+ },
+ cancel: () => reader.cancel(),
+ });
+ return new Response(stream, { headers });
+}
+
interface UploadRequest {
wants: string[];
haves: string[];
@@ -116,7 +182,7 @@
caps: Set<string>;
}
-function parseUploadRequest(body: Uint8Array): UploadRequest {
+export function parseUploadRequest(body: Uint8Array): UploadRequest {
const parser = new PktParser(body);
const req: UploadRequest = {
wants: [],
@@ -147,9 +213,9 @@
}
/** Follow tag objects down to the underlying commit oid (or null). */
-function peelToCommitOid(store: GitStore, oid: string): string | null {
+async function peelToCommitOid(store: GitStore, oid: string): Promise<string | null> {
for (let i = 0; i < 10; i++) {
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (!obj) return null;
if (obj.type === "commit") return oid;
if (obj.type === "tag") {
@@ -165,16 +231,16 @@
* Depth-limited commit set from the wants (BFS, min depth wins; the tip is
* depth 1, like git). Boundary commits are included but their parents cut.
*/
-function computeDepthSet(
+async function computeDepthSet(
store: GitStore,
wants: string[],
depth: number
-): { commits: Set<string>; boundary: Set<string> } {
+): Promise<{ commits: Set<string>; boundary: Set<string> }> {
const commits = new Set<string>();
const boundary = new Set<string>();
const queue: { oid: string; depth: number }[] = [];
for (const w of wants) {
- const c = peelToCommitOid(store, w);
+ const c = await peelToCommitOid(store, w);
if (c && !commits.has(c)) {
commits.add(c);
queue.push({ oid: c, depth: 1 });
@@ -182,7 +248,7 @@
}
while (queue.length) {
const { oid, depth: d } = queue.shift()!;
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (obj?.type !== "commit") continue;
const parents = parseCommit(obj.data).parents;
if (d >= depth) {
@@ -221,13 +287,13 @@
const set = new OidSet(4096);
const stack: string[] = [];
for (const w of wants) {
- const c = peelToCommitOid(store, w);
+ const c = await peelToCommitOid(store, w);
if (c && !haveCommits.has(c) && set.addHex(c)) stack.push(c);
}
let minTime = Infinity;
while (stack.length) {
const oid = stack.pop()!;
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (obj?.type !== "commit") continue;
await yieldMaybe();
const c = parseCommit(obj.data);
@@ -272,7 +338,7 @@
const haveCommits = new Set<string>();
for (const h of haves) {
if (!store.has(h)) continue;
- const c = peelToCommitOid(store, h);
+ const c = await peelToCommitOid(store, h);
if (c) haveCommits.add(c);
}
@@ -289,7 +355,7 @@
}
while (commitStack.length) {
const oid = commitStack.pop()!;
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (!obj) continue;
await yieldMaybe();
if (obj.type === "tag") {
@@ -314,7 +380,7 @@
for (let i = 0; i < interesting.size; i++) {
const oid = interesting.atHex(i);
if (excluded.hasHex(oid)) continue; // contested: reachable from a have too
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (obj?.type !== "commit") continue;
await yieldMaybe();
for (const p of parseCommit(obj.data).parents) {
@@ -324,7 +390,7 @@
const trees: string[] = [];
for (const oid of boundary) {
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (obj?.type !== "commit") continue;
const tree = parseCommit(obj.data).tree;
if (tree && excluded.addHex(tree)) trees.push(tree);
@@ -331,7 +397,7 @@
}
while (trees.length) {
const oid = trees.pop()!;
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (obj?.type !== "tree") continue;
await yieldMaybe();
for (const e of parseTree(obj.data)) {
@@ -377,7 +443,7 @@
if (meta.type === "commit" && commitLimit && !commitLimit.has(oid)) continue;
if (excluded.hasHex(oid)) {
if (descendThroughExcluded && meta.type === "commit") {
- const full = store.get(oid)!;
+ const full = (await store.get(oid))!;
stack.push(...parseCommit(full.data).parents);
}
continue;
@@ -384,7 +450,7 @@
}
walk.markHex(oid);
if (meta.type === "blob") continue; // leaf: membership only
- const full = store.get(oid)!;
+ const full = (await store.get(oid))!;
if (full.type === "commit") {
const c = parseCommit(full.data);
stack.push(c.tree, ...c.parents);
@@ -409,7 +475,8 @@
export async function uploadPack(
store: GitStore,
body: Uint8Array,
- release: () => void = () => {}
+ release: () => void = () => {},
+ packCache?: PackCache
): Promise<Response> {
const headers = {
"content-type": "application/x-git-upload-pack-result",
@@ -421,7 +488,7 @@
return new Response(pkt("ERR no valid wants\n") as unknown as BodyInit, { headers });
}
// a want must be an object we currently advertise, not merely one we store
- const allowed = advertisedOids(store);
+ const allowed = await advertisedOids(store);
for (const w of req.wants) {
if (!allowed.has(w) || !store.has(w)) {
release();
@@ -441,7 +508,7 @@
if (store.has(s)) preamble.push(pkt(`unshallow ${s}\n`));
}
} else {
- const { commits, boundary } = computeDepthSet(store, req.wants, req.deepen);
+ const { commits, boundary } = await computeDepthSet(store, req.wants, req.deepen);
commitLimit = commits;
for (const b of boundary) {
if (!clientShallowSet.has(b)) preamble.push(pkt(`shallow ${b}\n`));
@@ -496,6 +563,9 @@
return new Response(concat(preamble) as unknown as BodyInit, { headers });
}
+ const sideband = req.caps.has("side-band-64k");
+ const noProgress = req.caps.has("no-progress");
+
// Full-clone fast path: no haves, not shallow/deepen, and the wants are
// exactly the current ref tips. The reachable object set is then identical to
// what the graph walk would rediscover, so serve it from the versioned cache
@@ -504,6 +574,18 @@
// through to the walk, which then repopulates it — so a wrong pack is never
// served, and existing repos become fast on their next full clone.
const fullClone = !req.haves.length && req.deepen === 0 && wantsAreAllTips(store, req.wants);
+
+ // R2 full-clone offload (write side). Only a standard side-band-64k full clone
+ // is eligible, and it is reached only after the advertised-oids gate above — so
+ // the versioned key can never warm an unreachable/force-pushed object. The key
+ // is the ref version, so a push/force-push/delete lands on a fresh key and
+ // stale packs are simply never referenced. HITs are served by the Worker
+ // streaming R2 directly; the DO only reaches here on a MISS, where it serves
+ // the client and self-warms the cache via multipart while it streams (see
+ // `capture` below). Absent R2 (celld / no binding) this is a plain DO clone.
+ const r2Key =
+ fullClone && sideband && packCache ? `pack/${packCache.repo}/${store.reachableVersion()}` : null;
+
let send = fullClone ? store.loadReachable() : null;
if (!send) {
const excluded = await excludedObjects(store, req.haves, req.clientShallows, req.wants);
@@ -621,9 +703,6 @@
});
}
- const sideband = req.caps.has("side-band-64k");
- const noProgress = req.caps.has("no-progress");
-
// stream the pack: preamble, then pack bytes (side-band framed if negotiated)
const pending: Uint8Array[] = [concat(preamble)];
let buffered: Uint8Array[] = [];
@@ -636,7 +715,18 @@
if (sideband) pending.push(...sidebandFrames(1, payload));
else pending.push(payload);
};
+ // Self-warm R2 on a full-clone MISS: tee the raw pack (writer output, verbatim
+ // — same bytes a later HIT re-frames) into a multipart upload alongside the
+ // live stream. The pack streams into R2 in 16 MiB parts as it is generated, so
+ // isolate memory is never proportional to pack size and there is no size cap.
+ // A failed begin just streams the clone without caching.
+ let capture: MultipartCapture | null = null;
+ if (r2Key && packCache) {
+ const mp = await packCache.beginMultipart(r2Key, total);
+ if (mp) capture = new MultipartCapture(mp);
+ }
const writer = new PackWriter((chunk) => {
+ if (capture) capture.push(chunk);
buffered.push(chunk);
bufferedLen += chunk.length;
if (bufferedLen >= SIDEBAND_CHUNK) flushBuffered();
@@ -653,7 +743,7 @@
for (const c of pending) ctrl.enqueue(c);
pending.length = 0;
},
- pull: (ctrl) => {
+ pull: async (ctrl) => {
try {
for (let n = 0; n < 64 && i < total; n++, i++) {
const idx = order ? order[i] : i;
@@ -663,11 +753,11 @@
const oid = send.markedAtHex(idx);
const entry = store.packs.lookup(oid);
if (entry && !entry.baseOid) {
- writer.rawFull(entry.type, entry.entrySize, store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen));
+ writer.rawFull(entry.type, entry.entrySize, await store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen));
} else if (entry && entry.baseOid && send.isMarkedHex(entry.baseOid)) {
- writer.rawDelta(entry.entrySize, entry.baseOid, store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen));
+ writer.rawDelta(entry.entrySize, entry.baseOid, await store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen));
} else {
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (!obj) throw new Error(`missing object ${oid}`);
writer.object(obj.type, obj.data);
}
@@ -677,12 +767,12 @@
// the lean typed arrays, so no SQL lookup happens here.
const code = eCode[idx];
if (code <= -3) {
- writer.rawFull(NUM_TYPE[-2 - code], eSize![idx], store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx]));
+ writer.rawFull(NUM_TYPE[-2 - code], eSize![idx], await store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx]));
} else if (code >= 0) {
- writer.rawDelta(eSize![idx], send.atHex(code), store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx]));
+ writer.rawDelta(eSize![idx], send.atHex(code), await store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx]));
} else {
const oid = send.markedAtHex(idx);
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (!obj) throw new Error(`missing object ${oid}`);
writer.object(obj.type, obj.data);
}
@@ -697,17 +787,28 @@
}
for (const c of pending) ctrl.enqueue(c);
pending.length = 0;
+ // Warm R2 in lock-step with generation: the client already has this
+ // batch's bytes, and awaiting the part upload here is what bounds isolate
+ // memory to one part regardless of pack size. finish() flushes the final
+ // part and completes; a failure aborts, leaving no partial R2 object.
+ if (capture) {
+ if (finished) await capture.finish();
+ else await capture.drain();
+ }
if (finished) {
ctrl.close();
release(); // stream fully drained: this upload no longer holds a slot
}
} catch (err) {
+ if (capture) await capture.abort();
release();
ctrl.error(err);
}
},
- cancel: () => {
- // client went away mid-transfer (e.g. negotiation round abort)
+ cancel: async () => {
+ // client went away mid-transfer (e.g. negotiation round abort): abort the
+ // half-built multipart upload so R2 keeps no partial object
+ if (capture) await capture.abort();
release();
},
});
@@ -715,13 +816,13 @@
}
/** Is `anc` an ancestor of (or equal to) `desc`? Bounded walk. */
-export function isAncestor(store: GitStore, anc: string, desc: string): boolean {
+export async function isAncestor(store: GitStore, anc: string, desc: string): Promise<boolean> {
if (anc === desc) return true;
const seen = new Set<string>([desc]);
const stack = [desc];
let visited = 0;
while (stack.length && visited++ < 10000) {
- const obj = store.get(stack.pop()!);
+ const obj = await store.get(stack.pop()!);
if (obj?.type !== "commit") continue;
for (const p of parseCommit(obj.data).parents) {
if (p === anc) return true;
@@ -778,7 +879,7 @@
* closure was validated at their own push, so descent stops there — the walk
* touches only objects this push introduced.
*/
-function reachableComplete(store: GitStore, tip: string, known: OidSet): boolean {
+async function reachableComplete(store: GitStore, tip: string, known: OidSet): Promise<boolean> {
const stack = [tip];
while (stack.length) {
const oid = stack.pop()!;
@@ -786,7 +887,7 @@
const meta = store.typeAndSize(oid);
if (!meta) return false;
if (meta.type === "blob") continue;
- const obj = store.get(oid);
+ const obj = await store.get(oid);
if (!obj) return false;
if (obj.type === "commit") {
const c = parseCommit(obj.data);
@@ -803,15 +904,38 @@
return true;
}
-/** Apply ref updates after the pack (if any) has been ingested. */
-export function applyPushCommands(
+/** Per-command outcome decided by the object-reading validation pass. */
+interface PushDecision {
+ ref: string;
+ expectedOld: string; // CAS value re-checked atomically at commit
+ reject?: string; // ng message; the command does not apply
+ del?: boolean; // delete the ref
+ set?: string; // move the ref to this oid
+ forced?: boolean; // a set that strands old history (non-fast-forward)
+}
+
+/** Result of the async push validation, consumed by the sync commit under a savepoint. */
+export interface PushPlan {
+ decisions: PushDecision[];
+ known: OidSet;
+ hadCache: boolean;
+}
+
+/**
+ * Validate push commands against the (already ingested) object database. This
+ * is the object-reading half — connectivity + ancestry walks — and is async
+ * because an R2-backed pack's bytes are fetched over the network. It performs no
+ * writes; the ref mutations happen in commitPush under a savepoint. Splitting
+ * the read pass out is what lets the writes stay inside transactionSync (which
+ * cannot await) while still reading R2-backed objects. Pushes are serialized
+ * (receiveChain) and only pushes write refs, so ref state read here is stable
+ * through to commit; commitPush re-checks each CAS atomically regardless.
+ */
+export async function validatePush(
store: GitStore,
commands: PushCommand[],
unpackError: string | null
-): { results: CommandResult[]; changed: boolean; needsGc: boolean } {
- const results: CommandResult[] = [];
- let changed = false;
- let needsGc = false;
+): Promise<PushPlan> {
// whether the reachable cache is valid for the pre-push refs: only then can
// it be extended in place (union with the connectivity walk) instead of
// rebuilt from scratch on the next full clone
@@ -821,14 +945,16 @@
const known = new OidSet(4096);
for (const r of store.refs()) {
known.addHex(r.target);
- const c = peelToCommitOid(store, r.target);
+ const c = await peelToCommitOid(store, r.target);
if (c) known.addHex(c);
}
const knownHead = store.resolveHead();
if (knownHead) known.addHex(knownHead);
+
+ const decisions: PushDecision[] = [];
for (const cmd of commands) {
if (unpackError) {
- results.push({ ref: cmd.ref, ok: false, msg: "unpacker error" });
+ decisions.push({ ref: cmd.ref, expectedOld: cmd.old, reject: "unpacker error" });
continue;
}
const bad = !cmd.ref.startsWith("refs/") || cmd.ref.includes("..") || BAD_REF.test(cmd.ref)
@@ -836,29 +962,61 @@
|| cmd.ref.endsWith("/") || cmd.ref.length > 255
|| cmd.ref.split("/").some((c) => c.startsWith("."));
if (bad) {
- results.push({ ref: cmd.ref, ok: false, msg: "funny refname" });
+ decisions.push({ ref: cmd.ref, expectedOld: cmd.old, reject: "funny refname" });
continue;
}
const current = store.getRef(cmd.ref) ?? ZERO_OID;
if (current !== cmd.old) {
- results.push({ ref: cmd.ref, ok: false, msg: "fetch first" });
+ decisions.push({ ref: cmd.ref, expectedOld: cmd.old, reject: "fetch first" });
continue;
}
if (cmd.next === ZERO_OID) {
- store.delRef(cmd.ref);
+ decisions.push({ ref: cmd.ref, expectedOld: cmd.old, del: true });
+ continue;
+ }
+ if (!(await reachableComplete(store, cmd.next, known))) {
+ decisions.push({ ref: cmd.ref, expectedOld: cmd.old, reject: "missing necessary objects" });
+ continue;
+ }
+ const forced = cmd.old !== ZERO_OID && !(await isAncestor(store, cmd.old, cmd.next));
+ decisions.push({ ref: cmd.ref, expectedOld: cmd.old, set: cmd.next, forced });
+ }
+ return { decisions, known, hadCache };
+}
+
+/**
+ * Apply a validated PushPlan under a savepoint (transactionSync): a multi-ref
+ * push must not leave half its branches moved if a later update throws. Each CAS
+ * is re-checked here against live ref state so the write is atomic with its
+ * guard; a mismatch (a ref moved between validation and commit, which the push
+ * serialization normally prevents) both rejects that command and forces the
+ * reachable cache to be rebuilt rather than extended from a now-stale walk.
+ */
+export function commitPush(store: GitStore, plan: PushPlan): { results: CommandResult[]; changed: boolean; needsGc: boolean } {
+ const results: CommandResult[] = [];
+ let changed = false;
+ let needsGc = false;
+ let casMismatch = false;
+ for (const d of plan.decisions) {
+ if (d.reject) {
+ results.push({ ref: d.ref, ok: false, msg: d.reject });
+ continue;
+ }
+ const current = store.getRef(d.ref) ?? ZERO_OID;
+ if (current !== d.expectedOld) {
+ results.push({ ref: d.ref, ok: false, msg: "fetch first" });
+ casMismatch = true;
+ continue;
+ }
+ if (d.del) {
+ store.delRef(d.ref);
needsGc = true;
} else {
- if (!reachableComplete(store, cmd.next, known)) {
- results.push({ ref: cmd.ref, ok: false, msg: "missing necessary objects" });
- continue;
- }
- if (cmd.old !== ZERO_OID && !isAncestor(store, cmd.old, cmd.next)) {
- needsGc = true; // forced update strands old history
- }
- store.setRef(cmd.ref, cmd.next);
+ store.setRef(d.ref, d.set!);
+ if (d.forced) needsGc = true; // forced update strands old history
}
changed = true;
- results.push({ ref: cmd.ref, ok: true });
+ results.push({ ref: d.ref, ok: true });
}
// keep HEAD pointing at a branch that exists (first push wins, prefer main/master)
@@ -875,10 +1033,11 @@
// walk: the pre-existing tips plus every object this push newly connected. On
// a pure fast-forward with a valid prior cache, union closes over exactly the
// new reachable set. Anything else (a force-push/delete that strands history,
- // or no prior cache to extend) invalidates it, so the next full clone rebuilds
- // by walking rather than risk serving objects that are no longer reachable.
+ // no prior cache to extend, or a commit-time CAS mismatch that leaves `known`
+ // describing a command that did not apply) invalidates it, so the next full
+ // clone rebuilds by walking rather than risk serving unreachable objects.
if (changed) {
- if (hadCache && !needsGc) store.extendReachable(known);
+ if (plan.hadCache && !needsGc && !casMismatch) store.extendReachable(plan.known);
else store.invalidateReachable();
}
return { results, changed, needsGc };
diff --git a/src/git/store.ts b/src/git/store.ts
@@ -103,10 +103,19 @@
return { type: rows[0].type, data: inflate(packed) };
}
- get(oid: string): { type: ObjType; data: Uint8Array } | null {
- return this.getLoose(oid) ?? this.packs.getObject(oid, this.cache);
+ async get(oid: string): Promise<{ type: ObjType; data: Uint8Array } | null> {
+ return this.getLoose(oid) ?? (await this.packs.getObject(oid, this.cache));
}
+ /**
+ * Bind (or clear) the R2 raw-pack backend for this repo. Absent binding keeps
+ * every pack in SQLite (celld / Workers without the binding). Idempotent and
+ * cheap: the DO is per-repo, so this is set on each request from the repo name.
+ */
+ setR2(bucket: R2Bucket | undefined, repo: string): void {
+ this.packs.setR2(bucket, repo);
+ }
+
put(oid: string, type: ObjType, data: Uint8Array): void {
if (this.sql.exec("SELECT 1 FROM objects WHERE oid = ?", oid).toArray().length > 0) return;
const packed = deflate(data);
diff --git a/src/index.ts b/src/index.ts
@@ -2,7 +2,14 @@
import type { RepoInfo } from "./registry";
import { esc, age, layout, htmlResponse, errorPage } from "./ui/html";
import { CSS } from "./ui/style";
+import { parseUploadRequest, streamingCloneResponse } from "./git/protocol";
+import { isOid } from "./git/util";
+import { gunzipLimited } from "./git/zlib";
+import type { RepoCell } from "./repo";
+/** an upload-pack negotiation body is wants/haves/caps only — a few hundred KB */
+const MAX_UPLOAD_PACK_BYTES = 16 * 1024 * 1024;
+
export { RepoCell } from "./repo";
export { Registry } from "./registry";
@@ -193,6 +200,54 @@
}
}
+/**
+ * R2 full-clone fast path, served entirely from the Worker so the DO is never
+ * contacted for bytes (this is the read-offload that avoids the workerd
+ * DO->response stall on large packs). Detects a true full clone from the tiny
+ * negotiation body — wants, no haves, `done`, not shallow/deepen, side-band-64k
+ * — then asks the DO only for the cheap versioned key (no pack generation) and
+ * streams the stored raw pack from R2 with on-the-fly side-band framing. Returns
+ * null (a miss, a partial/shallow fetch, or any failure) to fall through to the
+ * normal DO clone path. Callers MUST invoke this only after the auth gate, so a
+ * private repo never streams R2 bytes to an anonymous client.
+ */
+async function serveCloneFromR2(
+ env: Env,
+ repo: string,
+ body: Uint8Array,
+ stub: DurableObjectStub<RepoCell>
+): Promise<Response | null> {
+ const bucket = env.PACK_CACHE;
+ if (!bucket) return null;
+ const req = parseUploadRequest(body);
+ const fullClone =
+ req.wants.length > 0 &&
+ req.wants.every(isOid) &&
+ req.haves.length === 0 &&
+ req.done &&
+ req.deepen === 0 &&
+ req.clientShallows.length === 0 &&
+ req.caps.has("side-band-64k");
+ if (!fullClone) return null;
+ let key: string | null;
+ try {
+ key = await stub.currentPackKey(repo, req.wants);
+ } catch {
+ return null;
+ }
+ if (!key) return null;
+ let obj: R2ObjectBody | null;
+ try {
+ obj = await bucket.get(key);
+ } catch {
+ return null;
+ }
+ if (!obj) return null;
+ const objects = parseInt(obj.customMetadata?.objects ?? "", 10);
+ if (!Number.isFinite(objects)) return null;
+ return streamingCloneResponse(obj.body, objects, req.caps.has("no-progress"));
+}
+
export default {
async fetch(req: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
const url = new URL(req.url);
@@ -286,7 +341,30 @@
// MB binary packs). Forward explicit bytes there; stream on workerd.
let fwdBody: BodyInit | null = null;
if (req.method !== "GET" && req.method !== "HEAD" && req.body) {
- fwdBody = pageCache() === null ? ((await req.arrayBuffer()) as ArrayBuffer) : req.body;
+ const declared = parseInt(req.headers.get("content-length") ?? "", 10);
+ const smallUploadPack =
+ sub === "/git-upload-pack" && (!Number.isFinite(declared) || declared <= MAX_UPLOAD_PACK_BYTES);
+ if (smallUploadPack) {
+ // buffer the tiny negotiation body so a full clone can be served straight
+ // from R2 (never touching the DO for bytes); the auth gate above has
+ // already run, so a private repo cannot reach this point anonymously.
+ const raw = new Uint8Array(await req.arrayBuffer());
+ let parsed: Uint8Array = raw;
+ if (req.headers.get("content-encoding")?.includes("gzip")) {
+ try {
+ parsed = gunzipLimited(raw, MAX_UPLOAD_PACK_BYTES);
+ } catch {
+ parsed = new Uint8Array(0);
+ }
+ }
+ if (env.PACK_CACHE && parsed.length) {
+ const hit = await serveCloneFromR2(env, repo, parsed, stub);
+ if (hit) return hit;
+ }
+ fwdBody = raw; // miss: forward the original (still-encoded) bytes to the DO
+ } else {
+ fwdBody = pageCache() === null ? ((await req.arrayBuffer()) as ArrayBuffer) : req.body;
+ }
}
const fwd = new Request(doUrl.toString(), {
method: req.method,
diff --git a/src/repo.ts b/src/repo.ts
@@ -9,10 +9,13 @@
advertisement,
uploadPack,
parsePushCommands,
- applyPushCommands,
+ validatePush,
+ commitPush,
renderStatus,
sidebandFrames,
+ wantsAreAllTips,
Service,
+ PackCache,
} from "./git/protocol";
import {
Commit,
@@ -95,6 +98,16 @@
this.store = new GitStore(ctx.storage.sql);
}
+ /**
+ * Bind the R2 raw-pack backend for this repo (absent binding keeps every pack
+ * in SQLite). The DO is per-repo, so the name is stable; persist it once so
+ * alarm-driven GC — which has no request to read x-repo from — can rebind.
+ */
+ private bindR2(repo: string): void {
+ this.store.setR2(this.env.PACK_CACHE, repo);
+ if (this.env.PACK_CACHE && this.store.getMeta("repo") !== repo) this.store.setMeta("repo", repo);
+ }
+
async fetch(req: Request): Promise<Response> {
const url = new URL(req.url);
const repo = req.headers.get("x-repo") ?? "repo";
@@ -101,6 +114,7 @@
const host = req.headers.get("x-host") ?? url.host;
const proto = req.headers.get("x-proto") ?? "https";
const path = url.pathname;
+ this.bindR2(repo);
try {
if (path === "/info/refs" && req.method === "GET") {
@@ -136,7 +150,7 @@
this.activeUploads--;
};
try {
- return await uploadPack(this.store, await this.readBody(req), release);
+ return await uploadPack(this.store, await this.readBody(req), release, this.packCacheFor(repo));
} catch (err) {
release();
// an over-inflating body is the client's fault, not a server error
@@ -168,7 +182,12 @@
// this instance stays resident: bring the (empty) schema back so
// later requests — including a re-creating push — find their tables
this.store = new GitStore(this.ctx.storage.sql);
+ this.bindR2(repo);
});
+ // the R2 packs (clone cache + raw pack store) outlive the SQLite wipe;
+ // purge them so a re-created repo never inherits stale bytes (its
+ // versioned keys would differ anyway)
+ await this.purgePackCache(repo);
return new Response("deleted\n");
}
if (path === "/gc" && req.method === "POST") {
@@ -176,7 +195,7 @@
}
if (req.method !== "GET") return new Response("method not allowed\n", { status: 405 });
- return this.ui(repo, host, proto, path, url.searchParams);
+ return await this.ui(repo, host, proto, path, url.searchParams);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return new Response(`error: ${msg}\n`, { status: 500 });
@@ -198,6 +217,9 @@
}
// clear first: at-most-once, so a throwing sweep can't trigger a retry storm
this.store.setMeta("gc-pending", "0");
+ // no request here to carry x-repo: rebind R2 from the persisted repo name so
+ // the sweep can read R2-backed objects and drop their raw objects
+ this.bindR2(this.store.getMeta("repo") ?? "repo");
try {
await this.gc();
} catch (err) {
@@ -214,7 +236,7 @@
let result: { removed: number; kept: number; skipped?: boolean } = { removed: 0, kept: 0 };
const run = this.receiveChain.then(() =>
this.ctx.blockConcurrencyWhile(async () => {
- result = this.runGc();
+ result = await this.runGc();
})
);
this.receiveChain = run.then(
@@ -248,7 +270,7 @@
* packs are kept (deleting mid-pack is impossible without a repack);
* huge repos skip the sweep entirely — the walk isn't worth it.
*/
- private runGc(): { removed: number; kept: number; skipped?: boolean } {
+ private async runGc(): Promise<{ removed: number; kept: number; skipped?: boolean }> {
if (this.store.objectCount() > 300_000) {
return { removed: 0, kept: this.store.objectCount(), skipped: true };
}
@@ -262,7 +284,7 @@
const meta = this.store.typeAndSize(oid);
if (!meta) continue;
if (meta.type === "blob") continue; // leaf: never inflate blob content here
- const obj = this.store.get(oid);
+ const obj = await this.store.get(oid);
if (!obj) continue;
if (obj.type === "commit") {
const c = parseCommit(obj.data);
@@ -296,7 +318,7 @@
for (let i = 0; i < reachable.size; i++) {
const oid = reachable.atHex(i);
if (this.store.packs.lookup(oid)) {
- const obj = this.store.get(oid);
+ const obj = await this.store.get(oid);
if (obj) {
this.store.put(oid, obj.type, obj.data);
migrated++;
@@ -304,7 +326,11 @@
}
}
removed += packCount - migrated;
+ // capture the R2-backed pack ids before the index is dropped, then delete
+ // their raw objects after reset so no orphaned bytes linger in R2
+ const r2ids = this.store.packs.r2PackIds();
this.store.packs.reset();
+ await this.store.packs.deleteR2Packs(r2ids);
}
// sweep loose objects (including anything just migrated) unreachable now
for (const oid of this.store.allOids()) {
@@ -316,6 +342,74 @@
return noHeadroom ? { removed, kept: reachable.size, skipped: true } : { removed, kept: reachable.size };
}
+ /**
+ * Cheap full-clone key probe for the Worker's R2 fast path: returns the
+ * versioned pack key iff `wants` is exactly the current ref tips (a true full
+ * clone). Reads ref state only — it never generates a pack. null (wants are
+ * not all tips) sends the Worker back to the normal DO clone path. The key is
+ * the ref version, so it can never name a stale/force-pushed pack.
+ */
+ currentPackKey(repo: string, wants: string[]): string | null {
+ if (!wantsAreAllTips(this.store, wants)) return null;
+ return `pack/${repo}/${this.store.reachableVersion()}`;
+ }
+
+ /**
+ * R2 full-clone offload adapter (write side), or undefined when no bucket is
+ * bound (celld and Workers without the binding both fall back to the pure DO
+ * path). A throwing createMultipartUpload (celld deliberately makes R2 throw)
+ * degrades to null so the clone still streams uncached; part-level failures
+ * are handled by MultipartCapture aborting the upload.
+ */
+ private packCacheFor(repo: string): PackCache | undefined {
+ const bucket = this.env.PACK_CACHE;
+ if (!bucket) return undefined;
+ return {
+ repo,
+ beginMultipart: async (key, objects) => {
+ try {
+ const mp = await bucket.createMultipartUpload(key, {
+ customMetadata: { objects: String(objects) },
+ });
+ const parts: R2UploadedPart[] = [];
+ return {
+ uploadPart: async (data) => {
+ parts.push(await mp.uploadPart(parts.length + 1, data));
+ },
+ complete: async () => {
+ await mp.complete(parts);
+ },
+ abort: async () => {
+ await mp.abort();
+ },
+ };
+ } catch (err) {
+ console.log(`[r2 ${repo}] multipart begin failed: ${err instanceof Error ? err.message : String(err)}`);
+ return null;
+ }
+ },
+ };
+ }
+
+ /** Purge every R2 object for a repo on delete: clone-cache packs (all versions)
+ * and raw pack-store objects. */
+ private async purgePackCache(repo: string): Promise<void> {
+ const bucket = this.env.PACK_CACHE;
+ if (!bucket) return;
+ for (const prefix of [`pack/${repo}/`, `raw/${repo}/`]) {
+ try {
+ for (let cursor: string | undefined; ; ) {
+ const listed = await bucket.list({ prefix, cursor });
+ if (listed.objects.length) await bucket.delete(listed.objects.map((o) => o.key));
+ if (!listed.truncated) break;
+ cursor = listed.cursor;
+ }
+ } catch (err) {
+ console.log(`[r2 ${repo}] purge ${prefix} failed: ${err instanceof Error ? err.message : String(err)}`);
+ }
+ }
+ }
+
private async readBody(req: Request): Promise<Uint8Array> {
let body: Uint8Array = new Uint8Array(await req.arrayBuffer());
if (body.length > MAX_UPLOAD_PACK_BYTES) throw new Error("request body exceeds maximum size");
@@ -433,6 +527,7 @@
cache: new ObjCache(budget),
onProgress: progress,
flush: () => this.ctx.storage.sync(),
+ collisionDetect: this.env.SHA1DC === "1",
});
} catch (err) {
unpackError = err instanceof Error ? err.message : String(err);
@@ -445,10 +540,13 @@
}
}
- // one savepoint around every ref update: a multi-ref push (git push --all)
- // must not leave half its branches moved if a later update throws
+ // validate connectivity/ancestry first (async: an R2-backed pack's objects
+ // are fetched over the network, which transactionSync cannot await), then
+ // apply the ref writes under one savepoint — a multi-ref push (git push
+ // --all) must not leave half its branches moved if a later update throws
+ const plan = await validatePush(this.store, commands, unpackError);
const { results, changed, needsGc } = this.ctx.storage.transactionSync(() =>
- applyPushCommands(this.store, commands, unpackError)
+ commitPush(this.store, plan)
);
if (changed) {
this.store.setMeta("created", "1");
@@ -474,7 +572,7 @@
}
- private base(repo: string, tab: string, ref?: string, formAction?: string): Omit<LayoutOpts, "body"> {
+ private async base(repo: string, tab: string, ref?: string, formAction?: string): Promise<Omit<LayoutOpts, "body">> {
const branches = this.store
.refs()
.filter((r) => r.name.startsWith("refs/heads/"))
@@ -489,13 +587,13 @@
sub: this.store.getMeta("description") || "[no description]",
tab,
ref: ref ?? headBranch,
- hasAbout: this.findReadme() !== null,
+ hasAbout: (await this.findReadme()) !== null,
branches,
formAction: formAction ?? `/${encodeURIComponent(repo)}/`,
};
}
- private ui(repo: string, host: string, proto: string, path: string, q: URLSearchParams): Response {
+ private async ui(repo: string, host: string, proto: string, path: string, q: URLSearchParams): Promise<Response> {
const h = q.get("h") ?? undefined;
if (path === "/" || path === "") return this.summaryPage(repo, host, proto);
if (path === "/about/" || path === "/about") return this.aboutPage(repo, h);
@@ -520,7 +618,7 @@
if (path.startsWith("/plain")) return this.plainPage(h, decodePath(path.slice("/plain".length)));
if (path.startsWith("/blame")) return this.blamePage(repo, h, decodePath(path.slice("/blame".length)));
if (path.startsWith("/snapshot/")) return this.snapshotPage(repo, decodeURIComponent(path.slice("/snapshot/".length)));
- return errorPage(this.base(repo, "summary"), `page not found: ${path}`);
+ return errorPage(await this.base(repo, "summary"), `page not found: ${path}`);
}
@@ -531,13 +629,13 @@
* brute-forced through the UI even after the protocol side stopped serving
* unreachable objects.
*/
- private resolveServable(id: string): string | null {
+ private async resolveServable(id: string): Promise<string | null> {
if (!isOid(id) || !this.store.has(id)) return null;
- return this.reachableOid(id) ? id : null;
+ return (await this.reachableOid(id)) ? id : null;
}
/** Is `target` reachable from any current ref (or HEAD)? */
- private reachableOid(target: string): boolean {
+ private async reachableOid(target: string): Promise<boolean> {
// the full-clone reachable-set cache is versioned against current refs
// (null on any mismatch), so a hit here is exactly as correct as the walk
// below and skips inflating/parsing the whole object graph
@@ -554,7 +652,7 @@
const meta = this.store.typeAndSize(oid);
if (!meta) continue;
if (meta.type === "blob") continue;
- const obj = this.store.get(oid);
+ const obj = await this.store.get(oid);
if (!obj) continue;
if (obj.type === "commit") {
const c = parseCommit(obj.data);
@@ -572,7 +670,7 @@
}
/** Resolve ?h= (branch, tag, full ref, or oid) to an object id. */
- private resolveRef(h?: string): { refName: string | null; oid: string } | null {
+ private async resolveRef(h?: string): Promise<{ refName: string | null; oid: string } | null> {
if (!h) {
const oid = this.store.resolveHead();
return oid ? { refName: this.store.head(), oid } : null;
@@ -581,7 +679,7 @@
const oid = this.store.getRef(cand);
if (oid) return { refName: cand, oid };
}
- const full = this.resolveServable(h);
+ const full = await this.resolveServable(h);
if (full) return { refName: null, oid: full };
return null;
}
@@ -591,15 +689,15 @@
* content, so a start oid that actually peeled through a tag has its final
* commit oid cached — a hit skips every intermediate tag object read.
*/
- private peelToCommit(oid: string): { oid: string; commit: Commit } | null {
+ private async peelToCommit(oid: string): Promise<{ oid: string; commit: Commit } | null> {
const start = oid;
const cached = this.store.getPeeled(start);
if (cached !== null) {
- const obj = this.store.get(cached);
+ const obj = await this.store.get(cached);
return obj?.type === "commit" ? { oid: cached, commit: parseCommit(obj.data) } : null;
}
for (let i = 0; i < 10; i++) {
- const obj = this.store.get(oid);
+ const obj = await this.store.get(oid);
if (!obj) return null;
if (obj.type === "commit") {
if (oid !== start) this.store.setPeeled(start, oid);
@@ -614,16 +712,16 @@
return null;
}
- private loadCommit(oid: string): Commit | null {
- const obj = this.store.get(oid);
+ private async loadCommit(oid: string): Promise<Commit | null> {
+ const obj = await this.store.get(oid);
return obj?.type === "commit" ? parseCommit(obj.data) : null;
}
/** oid of the entry at `path` in this commit's tree (any type), or null. */
- private pathOid(commit: Commit, path: string[]): string | null {
+ private async pathOid(commit: Commit, path: string[]): Promise<string | null> {
let oid = commit.tree;
for (const seg of path) {
- const obj = this.store.get(oid);
+ const obj = await this.store.get(oid);
if (obj?.type !== "tree") return null;
const entry = parseTree(obj.data).find((e) => e.name === seg);
if (!entry) return null;
@@ -633,12 +731,12 @@
}
/** pathOid keyed by the root tree it starts from: shared across a log walk. */
- private pathOidCached(treeOid: string, path: string[], memo: Map<string, string | null>): string | null {
+ private async pathOidCached(treeOid: string, path: string[], memo: Map<string, string | null>): Promise<string | null> {
const hit = memo.get(treeOid);
if (hit !== undefined) return hit;
let oid: string | null = treeOid;
for (const seg of path) {
- const obj = this.store.get(oid);
+ const obj = await this.store.get(oid);
if (obj?.type !== "tree") { oid = null; break; }
const entry = parseTree(obj.data).find((e) => e.name === seg);
if (!entry) { oid = null; break; }
@@ -648,11 +746,11 @@
return oid;
}
- private matchesFilter(e: LogEntry, filter: LogFilter, memo: Map<string, string | null>): boolean {
+ private async matchesFilter(e: LogEntry, filter: LogFilter, memo: Map<string, string | null>): Promise<boolean> {
if (filter.path && filter.path.length) {
- const mine = this.pathOidCached(e.commit.tree, filter.path, memo);
- const parentTree = e.commit.parents[0] ? this.loadCommit(e.commit.parents[0])?.tree : undefined;
- const theirs = parentTree ? this.pathOidCached(parentTree, filter.path, memo) : null;
+ const mine = await this.pathOidCached(e.commit.tree, filter.path, memo);
+ const parentTree = e.commit.parents[0] ? (await this.loadCommit(e.commit.parents[0]))?.tree : undefined;
+ const theirs = parentTree ? await this.pathOidCached(parentTree, filter.path, memo) : null;
if (mine === theirs) return false;
}
if (filter.q) {
@@ -670,8 +768,8 @@
}
/** Date-ordered commit walk (newest first) with optional filtering. */
- private walkLog(tip: string, skip: number, limit: number, filter: LogFilter = {}): { entries: LogEntry[]; more: boolean } {
- const first = this.peelToCommit(tip);
+ private async walkLog(tip: string, skip: number, limit: number, filter: LogFilter = {}): Promise<{ entries: LogEntry[]; more: boolean }> {
+ const first = await this.peelToCommit(tip);
if (!first) return { entries: [], more: false };
skip = Math.min(skip, MAX_LOG_SCAN); // a wild ofs must not drive the walk to its cap for an empty page
const scanCap = filter.path?.length ? PATH_LOG_SCAN : MAX_LOG_SCAN;
@@ -683,11 +781,11 @@
while (frontier.length && out.length < skip + limit + 1 && scanned++ < scanCap) {
frontier.sort((a, b) => b.commit.committer.time - a.commit.committer.time);
const cur = frontier.shift()!;
- if (this.matchesFilter(cur, filter, memo)) out.push(cur);
+ if (await this.matchesFilter(cur, filter, memo)) out.push(cur);
for (const p of cur.commit.parents) {
if (seen.has(p)) continue;
seen.add(p);
- const c = this.loadCommit(p);
+ const c = await this.loadCommit(p);
if (c) frontier.push({ oid: p, commit: c });
}
}
@@ -695,17 +793,17 @@
}
/** First-parent history of a path (for blame), newest first, with blobs. */
- private pathHistory(tip: string, path: string[], cap: number, maxBytes: number): BlameHistoryEntry[] {
+ private async pathHistory(tip: string, path: string[], cap: number, maxBytes: number): Promise<BlameHistoryEntry[]> {
const out: BlameHistoryEntry[] = [];
- let cur = this.peelToCommit(tip);
+ let cur = await this.peelToCommit(tip);
let steps = 0;
let bytes = 0;
while (cur && steps++ < MAX_LOG_SCAN && out.length < cap) {
- const myOid = this.pathOid(cur.commit, path);
- const parent = cur.commit.parents[0] ? this.peelToCommit(cur.commit.parents[0]) : null;
- const parentOid = parent ? this.pathOid(parent.commit, path) : null;
+ const myOid = await this.pathOid(cur.commit, path);
+ const parent = cur.commit.parents[0] ? await this.peelToCommit(cur.commit.parents[0]) : null;
+ const parentOid = parent ? await this.pathOid(parent.commit, path) : null;
if (myOid !== parentOid) {
- const blob = myOid ? this.store.get(myOid) : null;
+ const blob = myOid ? await this.store.get(myOid) : null;
const data = blob?.type === "blob" ? blob.data : null;
if (data) {
bytes += data.length;
@@ -720,7 +818,7 @@
}
/** Map oid -> decorations (branch/tag pointing at it). */
- private decorations(repo: string): Map<string, string> {
+ private async decorations(repo: string): Promise<Map<string, string>> {
const map = new Map<string, string>();
const r = `/${encodeURIComponent(repo)}`;
for (const ref of this.store.refs()) {
@@ -729,7 +827,7 @@
if (ref.name.startsWith("refs/heads/")) {
html = `<a class='branch-deco' href='${r}/log/?h=${encodeURIComponent(ref.name.slice(11))}'>${esc(ref.name.slice(11))}</a>`;
} else if (ref.name.startsWith("refs/tags/")) {
- const peeled = this.peelToCommit(ref.target);
+ const peeled = await this.peelToCommit(ref.target);
if (peeled) target = peeled.oid;
html = `<a class='tag-deco' href='${r}/tag/?h=${encodeURIComponent(ref.name.slice(10))}'>${esc(ref.name.slice(10))}</a>`;
} else continue;
@@ -738,13 +836,13 @@
return map;
}
- private lookupPath(
+ private async lookupPath(
rootTree: string,
path: string[]
- ): { kind: "tree"; entries: TreeEntry[] } | { kind: "blob"; entry: TreeEntry } | null {
+ ): Promise<{ kind: "tree"; entries: TreeEntry[] } | { kind: "blob"; entry: TreeEntry } | null> {
let treeOid = rootTree;
for (let i = 0; i < path.length; i++) {
- const obj = this.store.get(treeOid);
+ const obj = await this.store.get(treeOid);
if (obj?.type !== "tree") return null;
const entry = parseTree(obj.data).find((e) => e.name === path[i]);
if (!entry) return null;
@@ -754,17 +852,17 @@
if (!isTreeMode(entry.mode)) return null;
treeOid = entry.oid;
}
- const obj = this.store.get(treeOid);
+ const obj = await this.store.get(treeOid);
if (obj?.type !== "tree") return null;
return { kind: "tree", entries: parseTree(obj.data) };
}
- private findReadme(): { name: string; oid: string } | null {
+ private async findReadme(): Promise<{ name: string; oid: string } | null> {
const head = this.store.resolveHead();
if (!head) return null;
- const c = this.peelToCommit(head);
+ const c = await this.peelToCommit(head);
if (!c) return null;
- const root = this.store.get(c.commit.tree);
+ const root = await this.store.get(c.commit.tree);
if (root?.type !== "tree") return null;
const entries = parseTree(root.data);
for (const want of README_NAMES) {
@@ -774,22 +872,22 @@
return null;
}
- private flattenTree(treeOid: string, prefix: string, out: Map<string, { oid: string; mode: string }>): void {
- const obj = this.store.get(treeOid);
+ private async flattenTree(treeOid: string, prefix: string, out: Map<string, { oid: string; mode: string }>): Promise<void> {
+ const obj = await this.store.get(treeOid);
if (obj?.type !== "tree") return;
for (const e of parseTree(obj.data)) {
const p = prefix ? `${prefix}/${e.name}` : e.name;
- if (isTreeMode(e.mode)) this.flattenTree(e.oid, p, out);
+ if (isTreeMode(e.mode)) await this.flattenTree(e.oid, p, out);
else if (!isGitlinkMode(e.mode)) out.set(p, { oid: e.oid, mode: e.mode });
}
}
- private computeDiff(oldTree: string | null, newTree: string): { files: FileDiff[]; truncated: boolean } {
+ private async computeDiff(oldTree: string | null, newTree: string): Promise<{ files: FileDiff[]; truncated: boolean }> {
const oldFiles = new Map<string, { oid: string; mode: string }>();
const newFiles = new Map<string, { oid: string; mode: string }>();
- if (oldTree) this.flattenTree(oldTree, "", oldFiles);
- this.flattenTree(newTree, "", newFiles);
+ if (oldTree) await this.flattenTree(oldTree, "", oldFiles);
+ await this.flattenTree(newTree, "", newFiles);
const paths = [...new Set([...oldFiles.keys(), ...newFiles.keys()])].sort();
const files: FileDiff[] = [];
let truncated = false;
@@ -801,8 +899,8 @@
truncated = true;
break;
}
- const oldData = o ? this.store.get(o.oid)?.data ?? new Uint8Array(0) : new Uint8Array(0);
- const newData = n ? this.store.get(n.oid)?.data ?? new Uint8Array(0) : new Uint8Array(0);
+ const oldData = o ? (await this.store.get(o.oid))?.data ?? new Uint8Array(0) : new Uint8Array(0);
+ const newData = n ? (await this.store.get(n.oid))?.data ?? new Uint8Array(0) : new Uint8Array(0);
const fd: FileDiff = { path: p, o, n, kind: "text", ops: null, hunks: [], add: 0, del: 0 };
if (isBinary(oldData) || isBinary(newData)) {
fd.kind = "binary";
@@ -897,11 +995,11 @@
}
- private aboutPage(repo: string, h: string | undefined): Response {
- const base = this.base(repo, "about", h, `/${encodeURIComponent(repo)}/about/`);
- const readme = this.findReadme();
+ private async aboutPage(repo: string, h: string | undefined): Promise<Response> {
+ const base = await this.base(repo, "about", h, `/${encodeURIComponent(repo)}/about/`);
+ const readme = await this.findReadme();
if (!readme) return errorPage(base, "no readme found");
- const obj = this.store.get(readme.oid);
+ const obj = await this.store.get(readme.oid);
if (!obj) return errorPage(base, "missing readme blob");
const text = td.decode(obj.data);
const lower = readme.name.toLowerCase();
@@ -912,8 +1010,8 @@
return htmlResponse(layout({ ...base, body }));
}
- private summaryPage(repo: string, host: string, proto: string): Response {
- const base = this.base(repo, "summary");
+ private async summaryPage(repo: string, host: string, proto: string): Promise<Response> {
+ const base = await this.base(repo, "summary");
const branches = this.store.refs().filter((r) => r.name.startsWith("refs/heads/"));
const tags = this.store.refs().filter((r) => r.name.startsWith("refs/tags/"));
const r = `/${encodeURIComponent(repo)}`;
@@ -930,23 +1028,23 @@
);
}
- const branchRows = branches
+ const branchRows = (await Promise.all(branches
.slice(0, 10)
- .map((b) => {
+ .map(async (b) => {
const name = b.name.slice(11);
- const c = this.peelToCommit(b.target);
+ const c = await this.peelToCommit(b.target);
if (!c) return "";
return `<tr><td><a href='${r}/log/?h=${encodeURIComponent(name)}'>${esc(name)}</a></td>` +
`<td><a href='${r}/commit/?id=${c.oid}'>${esc(c.commit.subject)}</a></td>` +
`<td>${esc(c.commit.author.name)}</td><td>${age(c.commit.committer.time)}</td></tr>`;
- })
+ })))
.join("");
- const tagRows = tags
+ const tagRows = (await Promise.all(tags
.slice(0, 10)
- .map((t) => {
+ .map(async (t) => {
const name = t.name.slice(10);
- const obj = this.store.get(t.target);
+ const obj = await this.store.get(t.target);
let when = 0;
let target = t.target;
if (obj?.type === "tag") {
@@ -954,7 +1052,7 @@
when = tag.tagger?.time ?? 0;
target = tag.object;
}
- const c = this.peelToCommit(target);
+ const c = await this.peelToCommit(target);
if (c && !when) when = c.commit.committer.time;
const snap = `<a href='${r}/snapshot/${encodeURIComponent(repo)}-${encodeURIComponent(name)}.tar.gz'>tar.gz</a> ` +
`<a href='${r}/snapshot/${encodeURIComponent(repo)}-${encodeURIComponent(name)}.zip'>zip</a>`;
@@ -961,12 +1059,12 @@
return `<tr><td><a href='${r}/tag/?h=${encodeURIComponent(name)}'>${esc(name)}</a></td>` +
`<td><a href='${r}/commit/?id=${target}'>${esc(c?.commit.subject ?? "")}</a></td>` +
`<td>${esc(c?.commit.author.name ?? "")}</td><td>${age(when)}</td><td class='snapshots'>${snap}</td></tr>`;
- })
+ })))
.join("");
const headOid = this.store.resolveHead();
- const recent = headOid ? this.walkLog(headOid, 0, 10).entries : [];
- const deco = this.decorations(repo);
+ const recent = headOid ? (await this.walkLog(headOid, 0, 10)).entries : [];
+ const deco = await this.decorations(repo);
const logRows = recent
.map(
(e) =>
@@ -991,18 +1089,18 @@
return htmlResponse(layout({ ...base, body }));
}
- private logPage(repo: string, h: string | undefined, ofs: number, filter: LogFilter): Response {
+ private async logPage(repo: string, h: string | undefined, ofs: number, filter: LogFilter): Promise<Response> {
const r = `/${encodeURIComponent(repo)}`;
- const base = this.base(repo, "log", h, `${r}/log/`);
+ const base = await this.base(repo, "log", h, `${r}/log/`);
let tip = h;
if (filter.qt === "range" && filter.q) {
tip = filter.q;
filter = {};
}
- const rr = this.resolveRef(tip);
+ const rr = await this.resolveRef(tip);
if (!rr) return errorPage(base, tip ? `bad ref: ${tip}` : "empty repository");
- const { entries, more } = this.walkLog(rr.oid, ofs, LOG_PAGE, filter);
- const deco = this.decorations(repo);
+ const { entries, more } = await this.walkLog(rr.oid, ofs, LOG_PAGE, filter);
+ const deco = await this.decorations(repo);
const rows = entries
.map(
(e) =>
@@ -1049,27 +1147,27 @@
return htmlResponse(layout({ ...base, body }));
}
- private refsPage(repo: string): Response {
+ private async refsPage(repo: string): Promise<Response> {
const r = `/${encodeURIComponent(repo)}`;
- const base = this.base(repo, "refs", undefined, `${r}/refs/`);
+ const base = await this.base(repo, "refs", undefined, `${r}/refs/`);
const refs = this.store.refs();
const branches = refs.filter((x) => x.name.startsWith("refs/heads/"));
const tags = refs.filter((x) => x.name.startsWith("refs/tags/"));
- const branchRows = branches
- .map((b) => {
+ const branchRows = (await Promise.all(branches
+ .map(async (b) => {
const name = b.name.slice(11);
- const c = this.peelToCommit(b.target);
+ const c = await this.peelToCommit(b.target);
const snap = `<a href='${r}/snapshot/${encodeURIComponent(repo)}-${encodeURIComponent(name)}.tar.gz'>tar.gz</a> ` +
`<a href='${r}/snapshot/${encodeURIComponent(repo)}-${encodeURIComponent(name)}.zip'>zip</a>`;
return `<tr><td><a href='${r}/log/?h=${encodeURIComponent(name)}'>${esc(name)}</a></td>` +
`<td class='sha1'><a href='${r}/commit/?id=${b.target}'>${b.target.slice(0, 10)}</a></td>` +
`<td>${esc(c?.commit.author.name ?? "")}</td><td>${c ? age(c.commit.committer.time) : ""}</td><td class='snapshots'>${snap}</td></tr>`;
- })
+ })))
.join("");
- const tagRows = tags
- .map((t) => {
+ const tagRows = (await Promise.all(tags
+ .map(async (t) => {
const name = t.name.slice(10);
- const obj = this.store.get(t.target);
+ const obj = await this.store.get(t.target);
let when = 0;
let who = "";
let target = t.target;
@@ -1088,7 +1186,7 @@
return `<tr><td><a href='${r}/tag/?h=${encodeURIComponent(name)}'>${esc(name)}</a></td>` +
`<td class='sha1'><a href='${r}/commit/?id=${target}'>${target.slice(0, 10)}</a></td>` +
`<td>${esc(who)}</td><td>${age(when)}</td><td class='snapshots'>${snap}</td></tr>`;
- })
+ })))
.join("");
const body = `
<table class='list nowrap'>
@@ -1112,14 +1210,14 @@
return html;
}
- private treePage(repo: string, h: string | undefined, path: string[]): Response {
+ private async treePage(repo: string, h: string | undefined, path: string[]): Promise<Response> {
const r = `/${encodeURIComponent(repo)}`;
- const base = this.base(repo, "tree", h, `${r}/tree/${path.map(encodeURIComponent).join("/")}`);
- const rr = this.resolveRef(h);
+ const base = await this.base(repo, "tree", h, `${r}/tree/${path.map(encodeURIComponent).join("/")}`);
+ const rr = await this.resolveRef(h);
if (!rr) return errorPage(base, h ? `bad ref: ${h}` : "empty repository");
- const head = this.peelToCommit(rr.oid);
+ const head = await this.peelToCommit(rr.oid);
if (!head) return errorPage(base, "no commit found");
- const found = this.lookupPath(head.commit.tree, path);
+ const found = await this.lookupPath(head.commit.tree, path);
if (!found) return errorPage(base, `path not found: ${path.join("/")}`);
const withPath = { ...base, pathBar: this.pathBar(repo, h, path) };
@@ -1153,14 +1251,14 @@
return htmlResponse(layout({ ...withPath, body }));
}
- private blobPage(
+ private async blobPage(
repo: string,
h: string | undefined,
path: string[],
entry: TreeEntry,
base: Omit<LayoutOpts, "body">
- ): Response {
- const obj = this.store.get(entry.oid);
+ ): Promise<Response> {
+ const obj = await this.store.get(entry.oid);
if (!obj) return errorPage(base, "missing blob");
const r = `/${encodeURIComponent(repo)}`;
const q = h ? `?h=${encodeURIComponent(h)}` : "";
@@ -1182,22 +1280,22 @@
return htmlResponse(layout({ ...base, body }));
}
- private blamePage(repo: string, h: string | undefined, path: string[]): Response {
+ private async blamePage(repo: string, h: string | undefined, path: string[]): Promise<Response> {
const r = `/${encodeURIComponent(repo)}`;
- const base = this.base(repo, "tree", h, `${r}/blame/${path.map(encodeURIComponent).join("/")}`);
+ const base = await this.base(repo, "tree", h, `${r}/blame/${path.map(encodeURIComponent).join("/")}`);
const withPath = { ...base, pathBar: this.pathBar(repo, h, path) };
- const rr = this.resolveRef(h);
+ const rr = await this.resolveRef(h);
if (!rr) return errorPage(withPath, "empty repository");
if (!path.length) return errorPage(withPath, "blame needs a file path");
// size-gate the tip blob before walking history: a large file would
// otherwise load up to 200 full revisions into memory before blame() bails
- const tip = this.peelToCommit(rr.oid);
+ const tip = await this.peelToCommit(rr.oid);
if (!tip) return errorPage(withPath, "no commit found");
- const tipOid = this.pathOid(tip.commit, path);
+ const tipOid = await this.pathOid(tip.commit, path);
const meta = tipOid ? this.store.typeAndSize(tipOid) : null;
if (!meta || meta.type !== "blob") return errorPage(withPath, `no such file: ${path.join("/")}`);
if (meta.size > MAX_BLAME_BYTES) return errorPage(withPath, "blame skipped: file too large");
- const history = this.pathHistory(rr.oid, path, 200, MAX_BLAME_HISTORY_BYTES);
+ const history = await this.pathHistory(rr.oid, path, 200, MAX_BLAME_HISTORY_BYTES);
if (!history.length || !history[0].blob) return errorPage(withPath, `no such file: ${path.join("/")}`);
if (isBinary(history[0].blob)) return errorPage(withPath, "cannot blame a binary file");
const result = blame(history);
@@ -1223,37 +1321,37 @@
return htmlResponse(layout({ ...withPath, body }));
}
- private plainPage(h: string | undefined, path: string[]): Response {
- const rr = this.resolveRef(h);
+ private async plainPage(h: string | undefined, path: string[]): Promise<Response> {
+ const rr = await this.resolveRef(h);
if (!rr) return new Response("not found\n", { status: 404 });
- const head = this.peelToCommit(rr.oid);
+ const head = await this.peelToCommit(rr.oid);
if (!head) return new Response("not found\n", { status: 404 });
- const found = this.lookupPath(head.commit.tree, path);
+ const found = await this.lookupPath(head.commit.tree, path);
if (!found || found.kind !== "blob") return new Response("not found\n", { status: 404 });
- const obj = this.store.get(found.entry.oid);
+ const obj = await this.store.get(found.entry.oid);
if (!obj) return new Response("not found\n", { status: 404 });
return rawBlobResponse(obj.data, path[path.length - 1] ?? "");
}
- private blobByIdPage(id: string): Response {
- const oid = this.resolveServable(id);
+ private async blobByIdPage(id: string): Promise<Response> {
+ const oid = await this.resolveServable(id);
if (!oid) return new Response("not found\n", { status: 404 });
- const obj = this.store.get(oid);
+ const obj = await this.store.get(oid);
if (!obj || obj.type !== "blob") return new Response("not a blob\n", { status: 404 });
return rawBlobResponse(obj.data, "");
}
- private commitPage(repo: string, id: string | undefined, h: string | undefined): Response {
+ private async commitPage(repo: string, id: string | undefined, h: string | undefined): Promise<Response> {
const r = `/${encodeURIComponent(repo)}`;
- const base = this.base(repo, "commit", h, `${r}/commit/`);
- const oid = this.resolveCommitId(id, h);
+ const base = await this.base(repo, "commit", h, `${r}/commit/`);
+ const oid = await this.resolveCommitId(id, h);
if (!oid) return errorPage(base, "commit not found");
- const commit = this.loadCommit(oid);
+ const commit = await this.loadCommit(oid);
if (!commit) return errorPage(base, `commit not found: ${oid}`);
- const parent = commit.parents[0] ? this.loadCommit(commit.parents[0]) : null;
- const { files, truncated } = this.computeDiff(parent?.tree ?? null, commit.tree);
- const deco = this.decorations(repo);
+ const parent = commit.parents[0] ? await this.loadCommit(commit.parents[0]) : null;
+ const { files, truncated } = await this.computeDiff(parent?.tree ?? null, commit.tree);
+ const deco = await this.decorations(repo);
const body = `
<table class='commit-info'>
@@ -1270,30 +1368,30 @@
return htmlResponse(layout({ ...base, body }));
}
- private resolveCommitId(id: string | undefined, h: string | undefined): string | null {
+ private async resolveCommitId(id: string | undefined, h: string | undefined): Promise<string | null> {
if (id) {
- const full = this.resolveServable(id);
- return full ? this.peelToCommit(full)?.oid ?? null : null;
+ const full = await this.resolveServable(id);
+ return full ? (await this.peelToCommit(full))?.oid ?? null : null;
}
- const rr = this.resolveRef(h);
- return rr ? this.peelToCommit(rr.oid)?.oid ?? null : null;
+ const rr = await this.resolveRef(h);
+ return rr ? (await this.peelToCommit(rr.oid))?.oid ?? null : null;
}
/** diff/rawdiff: changes id2..id (default id2 = first parent of id). */
- private diffPage(repo: string, id: string | undefined, id2: string | undefined, h: string | undefined, raw: boolean): Response {
- const base = this.base(repo, "diff", h, `/${encodeURIComponent(repo)}/diff/`);
- const newOid = this.resolveCommitId(id, h);
+ private async diffPage(repo: string, id: string | undefined, id2: string | undefined, h: string | undefined, raw: boolean): Promise<Response> {
+ const base = await this.base(repo, "diff", h, `/${encodeURIComponent(repo)}/diff/`);
+ const newOid = await this.resolveCommitId(id, h);
if (!newOid) return raw ? new Response("not found\n", { status: 404 }) : errorPage(base, "commit not found");
- const commit = this.loadCommit(newOid)!;
+ const commit = (await this.loadCommit(newOid))!;
let oldOid: string | null = null;
if (id2) {
- oldOid = this.resolveCommitId(id2, undefined);
+ oldOid = await this.resolveCommitId(id2, undefined);
if (!oldOid) return raw ? new Response("bad id2\n", { status: 404 }) : errorPage(base, `bad id2: ${id2}`);
} else {
oldOid = commit.parents[0] ?? null;
}
- const oldCommit = oldOid ? this.loadCommit(oldOid) : null;
- const { files, truncated } = this.computeDiff(oldCommit?.tree ?? null, commit.tree);
+ const oldCommit = oldOid ? await this.loadCommit(oldOid) : null;
+ const { files, truncated } = await this.computeDiff(oldCommit?.tree ?? null, commit.tree);
if (raw) {
return new Response(this.renderRawDiff(files), { headers: { "content-type": "text/plain; charset=utf-8" } });
}
@@ -1304,12 +1402,12 @@
}
/** git-format-patch style output, applies with `git am`. */
- private patchPage(repo: string, id: string | undefined, h: string | undefined): Response {
- const oid = this.resolveCommitId(id, h);
+ private async patchPage(repo: string, id: string | undefined, h: string | undefined): Promise<Response> {
+ const oid = await this.resolveCommitId(id, h);
if (!oid) return new Response("not found\n", { status: 404 });
- const commit = this.loadCommit(oid)!;
- const parent = commit.parents[0] ? this.loadCommit(commit.parents[0]) : null;
- const { files } = this.computeDiff(parent?.tree ?? null, commit.tree);
+ const commit = (await this.loadCommit(oid))!;
+ const parent = commit.parents[0] ? await this.loadCommit(commit.parents[0]) : null;
+ const { files } = await this.computeDiff(parent?.tree ?? null, commit.tree);
const bodyText = commit.message.split("\n").slice(1).join("\n").trim();
let statLines = "";
let totalAdd = 0, totalDel = 0;
@@ -1334,13 +1432,13 @@
return new Response(patch, { headers: { "content-type": "text/plain; charset=utf-8" } });
}
- private tagPage(repo: string, name: string | undefined): Response {
+ private async tagPage(repo: string, name: string | undefined): Promise<Response> {
const r = `/${encodeURIComponent(repo)}`;
- const base = this.base(repo, "refs", undefined, `${r}/refs/`);
+ const base = await this.base(repo, "refs", undefined, `${r}/refs/`);
if (!name) return errorPage(base, "no tag given");
- const target = this.store.getRef(`refs/tags/${name}`) ?? this.resolveServable(name);
+ const target = this.store.getRef(`refs/tags/${name}`) ?? (await this.resolveServable(name));
if (!target) return errorPage(base, `tag not found: ${name}`);
- const obj = this.store.get(target);
+ const obj = await this.store.get(target);
if (!obj) return errorPage(base, `missing object`);
let body: string;
if (obj.type === "tag") {
@@ -1363,7 +1461,7 @@
return htmlResponse(layout({ ...base, body }));
}
- private snapshotPage(repo: string, filename: string): Response {
+ private async snapshotPage(repo: string, filename: string): Promise<Response> {
let format: "tar.gz" | "zip";
let stem: string;
if (filename.endsWith(".tar.gz")) {
@@ -1384,9 +1482,9 @@
for (const c of [...candidates]) candidates.push(`v${c}`);
let commit: { oid: string; commit: Commit } | null = null;
for (const cand of candidates) {
- const rr = this.resolveRef(cand);
+ const rr = await this.resolveRef(cand);
if (rr) {
- commit = this.peelToCommit(rr.oid);
+ commit = await this.peelToCommit(rr.oid);
if (commit) break;
}
}
@@ -1393,11 +1491,11 @@
if (!commit) return new Response(`no ref matches snapshot name: ${stem}\n`, { status: 404 });
const flat = new Map<string, { oid: string; mode: string }>();
- this.flattenTree(commit.commit.tree, "", flat);
+ await this.flattenTree(commit.commit.tree, "", flat);
const files: SnapshotFile[] = [];
let total = 0;
for (const [path, info] of flat) {
- const obj = this.store.get(info.oid);
+ const obj = await this.store.get(info.oid);
if (!obj) continue;
total += obj.data.length;
if (total > MAX_SNAPSHOT_BYTES) {
@@ -1422,10 +1520,10 @@
});
}
- private atomPage(repo: string, host: string, proto: string, h: string | undefined): Response {
- const rr = this.resolveRef(h);
+ private async atomPage(repo: string, host: string, proto: string, h: string | undefined): Promise<Response> {
+ const rr = await this.resolveRef(h);
if (!rr) return new Response("empty repository\n", { status: 404 });
- const { entries } = this.walkLog(rr.oid, 0, 20);
+ const { entries } = await this.walkLog(rr.oid, 0, 20);
const abs = `${proto}://${host}/${encodeURIComponent(repo)}`;
const iso = (t: number) => new Date(t * 1000).toISOString().replace(/\.\d+Z$/, "Z");
const updated = entries[0] ? iso(entries[0].commit.committer.time) : iso(0);
@@ -1460,13 +1558,13 @@
* History under an immutable commit oid never changes, so a cache hit
* needs no version check — only the pair itself as key.
*/
- private computeStats(
+ private async computeStats(
tipOid: string,
period: string
- ): { periodKeys: string[]; authors: { name: string; counts: Record<string, number> }[]; totals: Record<string, number>; count: number } {
+ ): Promise<{ periodKeys: string[]; authors: { name: string; counts: Record<string, number> }[]; totals: Record<string, number>; count: number }> {
const cached = this.store.getStatsCache(tipOid, period);
if (cached) return JSON.parse(cached);
- const { entries } = this.walkLog(tipOid, 0, MAX_STATS_SCAN);
+ const { entries } = await this.walkLog(tipOid, 0, MAX_STATS_SCAN);
const keyOf = (t: number): string => {
const d = new Date(t * 1000);
if (period === "y") return String(d.getUTCFullYear());
@@ -1504,12 +1602,12 @@
return result;
}
- private statsPage(repo: string, h: string | undefined, period: string): Response {
+ private async statsPage(repo: string, h: string | undefined, period: string): Promise<Response> {
const r = `/${encodeURIComponent(repo)}`;
- const base = this.base(repo, "stats", h, `${r}/stats/`);
- const rr = this.resolveRef(h);
+ const base = await this.base(repo, "stats", h, `${r}/stats/`);
+ const rr = await this.resolveRef(h);
if (!rr) return errorPage(base, "empty repository");
- const agg = this.computeStats(rr.oid, period);
+ const agg = await this.computeStats(rr.oid, period);
const cols = agg.periodKeys.slice(0, 8);
const authors = agg.authors
diff --git a/wrangler.jsonc b/wrangler.jsonc
@@ -22,6 +22,12 @@
"migrations": [
{ "tag": "v1", "new_sqlite_classes": ["RepoCell", "Registry"] }
],
+ // Optional: with PACK_CACHE bound, pushed pack bytes are stored in R2 (off the
+ // DO's SQLite) and full clones are served straight from R2 by the Worker.
+ // Remove this binding to fall back to the SQLite-only path (as on celld).
+ "r2_buckets": [
+ { "binding": "PACK_CACHE", "bucket_name": "dgit-pack-cache" }
+ ],
"vars": {
"SITE_NAME": "git.littledivy.com",
"SITE_DESC": "a fast webinterface for the git dscm, on Cloudflare",