| author | Divy Srivastava <me@littledivy.com> | 2026-08-19 08:30:58 +0530 |
|---|---|---|
| committer | Divy Srivastava <me@littledivy.com> | 2026-08-19 08:30:58 +0530 |
| commit | 0c364ef2f62ad36c5e72598973f2f44a17eb926c (patch) | |
| tree | eea793b50182c749bad591ff3fc2036627c44bef | |
| parent | 1af7b19b36d9720582af237d1bcfdd417fa9ce3e (diff) | |
| download | dgit-0c364ef2f6.tar.gz zip | |
v0.0.1
Diffstat
| .gitignore | +2 | -0 |
| scripts/gen-sha1dc-tables.mjs (new) | +118 | -0 |
| src/git/diff.ts | +13 | -0 |
| src/git/objects.ts | +58 | -14 |
| src/git/oidset.ts | +87 | -13 |
| src/git/pack.ts | +23 | -133 |
| src/git/packstore.ts | +147 | -30 |
| src/git/pktline.ts | +23 | -2 |
| src/git/protocol.ts | +452 | -47 |
| src/git/sha1.ts | +251 | -0 |
| src/git/sha1dc-tables.ts (new) | +322 | -0 |
| src/git/snapshot.ts | +20 | -2 |
| src/git/store.ts | +163 | -11 |
| src/git/zlib.ts | +19 | -15 |
| src/index.ts | +94 | -9 |
| src/registry.ts | +7 | -2 |
| src/repo.ts | +315 | -75 |
| src/ui/html.ts | +16 | -7 |
18 files changed, 2130 insertions(+), 360 deletions(-)
diff --git a/.gitignore b/.gitignore @@ -2,3 +2,5 @@ .wrangler/ .dev.vars dist/ +scripts/* +!scripts/gen-sha1dc-tables.mjs diff --git a/scripts/gen-sha1dc-tables.mjs b/scripts/gen-sha1dc-tables.mjs @@ -0,0 +1,118 @@ +#!/usr/bin/env node +/** + * Regenerates src/git/sha1dc-tables.ts from the upstream C source. + * + * node scripts/gen-sha1dc-tables.mjs [path/to/ubc_check.c] + * + * With no argument the file is fetched from cr-marcstevens/sha1collisiondetection. + * The disturbance-vector table and the unavoidable-bit-condition expressions + * are transcribed by rule, never retyped — a typo in 2560 table words would + * be undetectable by reading. Re-run scripts/sha1dc-test.mjs afterwards; it + * pins digests of both the DV table and ubcCheck's behaviour. + */ +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const UPSTREAM = + "https://raw.githubusercontent.com/cr-marcstevens/sha1collisiondetection/master/lib/ubc_check.c"; +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const OUT = join(ROOT, "src", "git", "sha1dc-tables.ts"); + +const src = process.argv[2] + ? readFileSync(process.argv[2], "utf8") + : await (await fetch(UPSTREAM)).text(); + +const bitConsts = new Map(); +for (const m of src.matchAll(/static const uint32_t (DV_\w+_bit)\s*=\s*\(uint32_t\)\(1\)\s*<<\s*(\d+);/g)) { + bitConsts.set(m[1], Number(m[2])); +} +if (bitConsts.size !== 32) throw new Error(`expected 32 DV bit constants, got ${bitConsts.size}`); + +const tableBody = src.slice(src.indexOf("dv_info_t sha1_dvs[] =")); +const rowRe = /\{\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*\{([^}]*)\}\s*\}/g; +const rows = []; +for (const m of tableBody.matchAll(rowRe)) { + const [, dvType, dvK, dvB, testt, maski, maskb] = m.map(Number); + if (dvType === 0) break; // sentinel row terminates the table + const dm = m[7].split(",").map((s) => s.trim()).filter(Boolean); + if (dm.length !== 80) throw new Error(`dm has ${dm.length} words, expected 80`); + rows.push({ dvType, dvK, dvB, testt, maski, maskb, dm }); +} +if (rows.length !== 32) throw new Error(`expected 32 disturbance vectors, got ${rows.length}`); +for (const r of rows) { + // ubc_check.h only defines DOSTORESTATE58 and DOSTORESTATE65, and + // DVMASKSIZE is 1, so the port keeps exactly two retained states and a + // single mask word. A new upstream table could break both assumptions. + if (r.testt !== 58 && r.testt !== 65) throw new Error(`unsupported testt ${r.testt}`); + if (r.maski !== 0) throw new Error(`unsupported maski ${r.maski}`); +} + +const fnStart = src.indexOf("void ubc_check(const uint32_t W[80], uint32_t dvmask[1])"); +if (fnStart < 0) throw new Error("ubc_check definition not found"); +let body = src.slice(src.indexOf("{", fnStart) + 1); +body = body.slice(0, body.lastIndexOf("dvmask[0]=mask;")); +body = body.replace(/uint32_t mask = ~\(\(uint32_t\)\(0\)\);/, ""); + +// C uint32 arithmetic maps onto JS int32 bit-for-bit: +// * '>>' on uint32 is a logical shift -> '>>>' +// * subtraction wraps mod 2^32; two's complement gives the same bits, and +// every result is consumed by a bitwise operator, so no masking is needed +// * C truthiness of 'if (x)' and '!x' matches JS for int32 (0 is falsy) +let js = body.replace(/>>/g, ">>>").replace(/DV_(\w+)_bit/g, (_, name) => { + const key = `DV_${name}_bit`; + if (!bitConsts.has(key)) throw new Error(`unknown constant ${key}`); + return `(1 << ${bitConsts.get(key)})`; +}); +for (const bad of ["uint32_t", "->", "sizeof", "static", "const "]) { + if (js.includes(bad)) throw new Error(`unconverted C construct: ${bad}`); +} +const stmts = js.split("\n").map((l) => l.trim()).filter(Boolean); +for (const s of stmts) { + if (!/^(mask &=|if \(|\|\||!|\)|\{|\}|else)/.test(s)) throw new Error(`unexpected statement: ${s}`); +} + +const dvLines = rows + .map( + (r) => + ` { testt: ${r.testt}, maskb: ${r.maskb}, dm: [${r.dm.join(",")}] },` + + ` // ${r.dvType === 1 ? "I" : "II"}(${r.dvK},${r.dvB})` + ) + .join("\n"); + +writeFileSync( + OUT, + `// GENERATED — do not edit by hand. +// Transcribed from sha1collisiondetection/lib/ubc_check.c (Marc Stevens, Dan +// Shumow; MIT). Regenerate with scripts/gen-sha1dc-tables.mjs. +// +// C uint32 arithmetic maps onto JS int32 bit-for-bit: '>>' becomes '>>>' and +// wrapping subtraction is left to two's-complement, since every result feeds a +// bitwise operator. + +/** One disturbance vector: the step to recompress from, its bit in the UBC + * mask, and the expanded-message XOR difference. */ +export interface Sha1Dv { + readonly testt: number; + readonly maskb: number; + readonly dm: readonly number[]; +} + +/** The ${rows.length} disturbance vectors checked by SHA-1DC. */ +export const SHA1_DVS: readonly Sha1Dv[] = [ +${dvLines} +]; + +/** + * Check the unavoidable bit conditions for every DV against an expanded + * message block. Returns a mask whose bit \`maskb\` is set when every UBC for + * that DV holds, i.e. when the DV is worth the cost of a recompression check. + */ +export function ubcCheck(W: Int32Array): number { + let mask = ~0; +${js.replace(/\n\t/g, "\n ").replace(/\t/g, " ").trimEnd()} + return mask; +} +` +); +console.log(`wrote ${OUT}: ${rows.length} DVs, ${stmts.length} UBC statements`); diff --git a/src/git/diff.ts b/src/git/diff.ts @@ -12,6 +12,9 @@ } const MAX_LINES = 40000; +const MAX_PRODUCT = 4_000_000; +const MAX_D = 2000; +const TRACE_BUDGET = 8_000_000; /** Myers O(ND) line diff. Returns null when the input is too large. */ export function diffLines(aText: string, bText: string): DiffOp[] | null { @@ -20,14 +23,24 @@ if (a[a.length - 1] === "") a.pop(); if (b[b.length - 1] === "") b.pop(); const N = a.length, M = b.length; + + if (N === 0 || M === 0) { + const ops: DiffOp[] = []; + for (const line of a) ops.push({ tag: "del", line }); + for (const line of b) ops.push({ tag: "add", line }); + return ops; + } if (N + M > MAX_LINES) return null; + if (N * M > MAX_PRODUCT) return null; const max = N + M; + const dCap = Math.min(MAX_D, Math.max(1, Math.floor(TRACE_BUDGET / max))); const offset = max; const v = new Int32Array(2 * max + 2); const trace: Int32Array[] = []; let dFound = -1; outer: for (let d = 0; d <= max; d++) { + if (d > dCap) return null; trace.push(v.slice()); for (let k = -d; k <= d; k += 2) { let x: number; diff --git a/src/git/objects.ts b/src/git/objects.ts @@ -1,7 +1,15 @@ -import { te, td, toHex, concat, sha1hex } from "./util"; +import { te, td, toHex, concat, sha1hex, isOid } from "./util"; export type ObjType = "commit" | "tree" | "blob" | "tag"; +/** Thrown for any malformed/attacker-crafted object body. */ +export class GitParseError extends Error { + constructor(msg: string) { + super(msg); + this.name = "GitParseError"; + } +} + export const TYPE_NUM: Record<ObjType, number> = { commit: 1, tree: 2, blob: 3, tag: 4 }; export const NUM_TYPE: Record<number, ObjType> = { 1: "commit", 2: "tree", 3: "blob", 4: "tag" }; @@ -46,10 +54,18 @@ } function parsePerson(line: string): Person { - // "Name <email> 1234567890 +0100" - const m = line.match(/^(.*?) <(.*?)> (\d+) ([+-]\d{4})$/); - if (!m) return { name: line, email: "", time: 0, tz: "+0000" }; - return { name: m[1], email: m[2], time: parseInt(m[3], 10), tz: m[4] }; + // "Name <email> 1234567890 +0100". Parsed with linear scans, not a + // backtracking regex: an attacker controls this line and can make it + // megabytes long, so `/(.*?) <(.*?)>/` would be a quadratic-scan DoS. + const lt = line.indexOf(" <"); + const gt = lt < 0 ? -1 : line.indexOf(">", lt + 2); + if (lt < 0 || gt < 0) return { name: line, email: "", time: 0, tz: "+0000" }; + const name = line.slice(0, lt); + const email = line.slice(lt + 2, gt); + const rest = line.slice(gt + 1).trim().split(" "); + const time = rest.length ? parseInt(rest[0], 10) : 0; + const tz = rest.length > 1 && /^[+-]\d{4}$/.test(rest[1]) ? rest[1] : "+0000"; + return { name, email, time: Number.isFinite(time) ? time : 0, tz }; } function splitHeaders(text: string): { headers: [string, string][]; message: string } { @@ -57,15 +73,25 @@ const head = nn === -1 ? text : text.slice(0, nn); const message = nn === -1 ? "" : text.slice(nn + 2); const headers: [string, string][] = []; + // continuation pieces are collected per-header and joined once at the end: + // appending with `+=` in the loop is O(n^2) on a crafted object with a huge + // multi-line header value (e.g. a giant gpgsig). + const cont: string[][] = []; for (const line of head.split("\n")) { // continuation lines (gpgsig etc.) start with a space; append to previous if (line.startsWith(" ") && headers.length) { - headers[headers.length - 1][1] += "\n" + line.slice(1); + cont[cont.length - 1].push(line.slice(1)); continue; } const sp = line.indexOf(" "); - if (sp > 0) headers.push([line.slice(0, sp), line.slice(sp + 1)]); + if (sp > 0) { + headers.push([line.slice(0, sp), line.slice(sp + 1)]); + cont.push([]); + } } + for (let i = 0; i < headers.length; i++) { + if (cont[i].length) headers[i][1] += "\n" + cont[i].join("\n"); + } return { headers, message }; } @@ -79,12 +105,22 @@ message, subject: message.split("\n", 1)[0] ?? "", }; + let sawTree = false; for (const [k, v] of headers) { - if (k === "tree") c.tree = v; - else if (k === "parent") c.parents.push(v); - else if (k === "author") c.author = parsePerson(v); + if (k === "tree") { + // tree/parent oids feed the connectivity walk verbatim: reject anything + // not a real oid so a crafted value can never be treated as an object id. + if (sawTree) throw new GitParseError("commit has duplicate tree header"); + if (!isOid(v)) throw new GitParseError("commit has malformed tree oid"); + c.tree = v; + sawTree = true; + } else if (k === "parent") { + if (!isOid(v)) throw new GitParseError("commit has malformed parent oid"); + c.parents.push(v); + } else if (k === "author") c.author = parsePerson(v); else if (k === "committer") c.committer = parsePerson(v); } + if (!sawTree) throw new GitParseError("commit missing tree header"); return c; } @@ -91,12 +127,18 @@ export function parseTag(data: Uint8Array): Tag { const { headers, message } = splitHeaders(td.decode(data)); const t: Tag = { object: "", type: "", tag: "", tagger: null, message }; + let sawObject = false; for (const [k, v] of headers) { - if (k === "object") t.object = v; - else if (k === "type") t.type = v; + if (k === "object") { + if (sawObject) throw new GitParseError("tag has duplicate object header"); + if (!isOid(v)) throw new GitParseError("tag has malformed object oid"); + t.object = v; + sawObject = true; + } else if (k === "type") t.type = v; else if (k === "tag") t.tag = v; else if (k === "tagger") t.tagger = parsePerson(v); } + if (!sawObject) throw new GitParseError("tag missing object header"); return t; } @@ -105,10 +147,12 @@ let pos = 0; while (pos < data.length) { let sp = pos; - while (data[sp] !== 0x20) sp++; + while (sp < data.length && data[sp] !== 0x20) sp++; + if (sp >= data.length) throw new GitParseError("malformed tree"); const mode = td.decode(data.subarray(pos, sp)); let nul = sp + 1; - while (data[nul] !== 0) nul++; + while (nul < data.length && data[nul] !== 0) nul++; + if (nul >= data.length || nul + 21 > data.length) throw new GitParseError("malformed tree"); const name = td.decode(data.subarray(sp + 1, nul)); const oid = toHex(data.subarray(nul + 1, nul + 21)); entries.push({ mode, name, oid }); diff --git a/src/git/oidset.ts b/src/git/oidset.ts @@ -5,6 +5,11 @@ * Set<string> of 40-char hex strings costs ~100 bytes per entry — at * Linux scale (10M objects) that is gigabytes. This stores raw 20-byte * digests in typed arrays: ~20B per entry plus a u32 open-addressing table. + * + * A set can also carry a *tagged sub-set* (mark/isMarked/markedAt): one bit + * per entry plus a u32 index list. Two nested roles — e.g. the object walk's + * "visited" and its strict subset "send" — then cost ~28B/object in one set + * instead of ~56B in two. */ export class OidSet { private table: Uint32Array; // 1-based indices into the entry list; 0 = empty @@ -11,6 +16,10 @@ private mask: number; private data: Uint8Array; // 20 bytes per entry, insertion order private count = 0; + // tagged sub-set, allocated lazily on first mark() + private marks: Uint8Array | null = null; // one bit per entry index + private markedIdx: Uint32Array | null = null; // tagged entry indices, insertion order + private markedCount = 0; constructor(expected = 1024) { let cap = 2048; @@ -24,6 +33,11 @@ return this.count; } + /** Number of entries in the tagged sub-set. */ + get markedSize(): number { + return this.markedCount; + } + private hashAt(bytes: Uint8Array, off: number): number { // oids are uniformly random; the first 4 bytes are a fine hash return ((bytes[off] << 24) | (bytes[off + 1] << 16) | (bytes[off + 2] << 8) | bytes[off + 3]) >>> 0; @@ -49,39 +63,66 @@ this.mask = newMask; } - /** Adds a hex oid; returns false if it was already present. */ - addHex(hex: string): boolean { - return this.addBytes(fromHex(hex), 0); + /** Make room for the entry about to be written at index `this.count`. */ + private growData(): void { + if ((this.count + 1) * 20 > this.data.length) { + const bigger = new Uint8Array(this.data.length * 2); + bigger.set(this.data); + this.data = bigger; + } + if (this.marks && (this.count >> 3) >= this.marks.length) { + const bigger = new Uint8Array(this.marks.length * 2); + bigger.set(this.marks); + this.marks = bigger; + } } - addBytes(bytes: Uint8Array, off: number): boolean { + /** Entry index of `bytes`, inserting it when absent. */ + private indexOrInsert(bytes: Uint8Array, off: number): number { if ((this.count + 1) * 2 > this.table.length) this.grow(); let slot = this.hashAt(bytes, off) & this.mask; while (this.table[slot] !== 0) { - if (this.equalsEntry(this.table[slot] - 1, bytes, off)) return false; + if (this.equalsEntry(this.table[slot] - 1, bytes, off)) return this.table[slot] - 1; slot = (slot + 1) & this.mask; } - if ((this.count + 1) * 20 > this.data.length) { - const bigger = new Uint8Array(this.data.length * 2); - bigger.set(this.data); - this.data = bigger; - } + this.growData(); this.data.set(bytes.subarray(off, off + 20), this.count * 20); this.table[slot] = ++this.count; - return true; + return this.count - 1; } + /** Adds a hex oid; returns false if it was already present. */ + addHex(hex: string): boolean { + return this.addBytes(fromHex(hex), 0); + } + + addBytes(bytes: Uint8Array, off: number): boolean { + const before = this.count; + // a freshly inserted entry always lands at index `before` + return this.indexOrInsert(bytes, off) === before; + } + hasHex(hex: string): boolean { return this.hasBytes(fromHex(hex), 0); } + /** Insertion-order entry index of `hex`, or -1 if absent. */ + indexOfHex(hex: string): number { + return this.indexOf(fromHex(hex), 0); + } + hasBytes(bytes: Uint8Array, off: number): boolean { + return this.indexOf(bytes, off) >= 0; + } + + /** Entry index of `bytes`, or -1. */ + private indexOf(bytes: Uint8Array, off: number): number { let slot = this.hashAt(bytes, off) & this.mask; while (this.table[slot] !== 0) { - if (this.equalsEntry(this.table[slot] - 1, bytes, off)) return true; + if (this.equalsEntry(this.table[slot] - 1, bytes, off)) return this.table[slot] - 1; slot = (slot + 1) & this.mask; } - return false; + return -1; } /** Hex oid of the i-th inserted entry. */ @@ -88,4 +129,37 @@ atHex(i: number): string { return toHex(this.data.subarray(i * 20, i * 20 + 20)); } + + /** + * Adds `hex` if needed and puts it in the tagged sub-set. + * Returns false if it was already tagged. + */ + markHex(hex: string): boolean { + const idx = this.indexOrInsert(fromHex(hex), 0); + if (!this.marks) this.marks = new Uint8Array(((this.data.length / 20) >> 3) + 1); + const byte = idx >> 3; + const bit = 1 << (idx & 7); + if (this.marks[byte] & bit) return false; + this.marks[byte] |= bit; + if (!this.markedIdx) this.markedIdx = new Uint32Array(1024); + if (this.markedCount === this.markedIdx.length) { + const bigger = new Uint32Array(this.markedIdx.length * 2); + bigger.set(this.markedIdx); + this.markedIdx = bigger; + } + this.markedIdx[this.markedCount++] = idx; + return true; + } + + isMarkedHex(hex: string): boolean { + if (!this.marks) return false; + const idx = this.indexOf(fromHex(hex), 0); + return idx >= 0 && (this.marks[idx >> 3] & (1 << (idx & 7))) !== 0; + } + + /** Hex oid of the i-th entry of the tagged sub-set. */ + markedAtHex(i: number): string { + const idx = this.markedIdx![i]; + return toHex(this.data.subarray(idx * 20, idx * 20 + 20)); + } } diff --git a/src/git/pack.ts b/src/git/pack.ts @@ -1,31 +1,19 @@ -import { te, td, toHex, fromHex, concat, sha1hex, Sha1 } from "./util"; -import { deflate, inflateEntry } from "./zlib"; -import { ObjType, NUM_TYPE, TYPE_NUM, objectHeader, hashObject } from "./objects"; +import { te, fromHex, Sha1 } from "./util"; +import { deflate } from "./zlib"; +import { ObjType, TYPE_NUM } from "./objects"; -export interface PackedObject { - oid: string; - type: ObjType; - data: Uint8Array; -} - -const OFS_DELTA = 6; -const REF_DELTA = 7; - -interface RawEntry { - offset: number; // offset of the entry header within the pack (ofs-delta base key) - type: number; - data: Uint8Array; // object content, or delta payload for delta entries - baseOffset?: number; - baseOid?: string; - resolved?: { type: ObjType; data: Uint8Array }; -} - export function applyDelta(base: Uint8Array, delta: Uint8Array): Uint8Array { let pos = 0; + // bounded read: a truncated/crafted delta must throw, never read past the end + // (which would fold `undefined` into offsets/sizes and corrupt the result). + const next = (): number => { + if (pos >= delta.length) throw new Error("delta truncated"); + return delta[pos++]; + }; const varint = () => { let r = 0, shift = 0, b: number; do { - b = delta[pos++]; + b = next(); r += (b & 0x7f) * 2 ** shift; shift += 7; } while (b & 0x80); @@ -34,25 +22,30 @@ const srcSize = varint(); const tgtSize = varint(); if (srcSize !== base.length) throw new Error("delta base size mismatch"); + if (tgtSize > 512 * 1024 * 1024) throw new Error("delta target too large"); const out = new Uint8Array(tgtSize); let op = 0; while (pos < delta.length) { - const cmd = delta[pos++]; + const cmd = next(); if (cmd & 0x80) { // copy from base let off = 0, size = 0; - if (cmd & 0x01) off = delta[pos++]; - if (cmd & 0x02) off |= delta[pos++] << 8; - if (cmd & 0x04) off |= delta[pos++] << 16; - if (cmd & 0x08) off += delta[pos++] * 0x1000000; - if (cmd & 0x10) size = delta[pos++]; - if (cmd & 0x20) size |= delta[pos++] << 8; - if (cmd & 0x40) size |= delta[pos++] << 16; + if (cmd & 0x01) off = next(); + if (cmd & 0x02) off |= next() << 8; + if (cmd & 0x04) off |= next() << 16; + if (cmd & 0x08) off += next() * 0x1000000; + if (cmd & 0x10) size = next(); + if (cmd & 0x20) size |= next() << 8; + if (cmd & 0x40) size |= next() << 16; if (size === 0) size = 0x10000; + if (off + size > base.length) throw new Error("delta copy out of range"); + if (op + size > tgtSize) throw new Error("delta copy overflows target"); out.set(base.subarray(off, off + size), op); op += size; } else if (cmd) { // insert literal + if (pos + cmd > delta.length) throw new Error("delta literal truncated"); + if (op + cmd > tgtSize) throw new Error("delta literal overflows target"); out.set(delta.subarray(pos, pos + cmd), op); op += cmd; pos += cmd; @@ -64,99 +57,6 @@ return out; } -/** - * Parse and index a packfile: inflate every entry, resolve ofs/ref deltas - * (including thin-pack bases fetched through `getBase`), and hash each object. - */ -export function indexPack( - pack: Uint8Array, - getBase: (oid: string) => { type: ObjType; data: Uint8Array } | null -): PackedObject[] { - if (pack.length < 32 || td.decode(pack.subarray(0, 4)) !== "PACK") - throw new Error("bad pack signature"); - const view = new DataView(pack.buffer, pack.byteOffset, pack.byteLength); - const version = view.getUint32(4); - if (version !== 2 && version !== 3) throw new Error(`unsupported pack version ${version}`); - const count = view.getUint32(8); - - // verify trailer checksum - const trailer = toHex(pack.subarray(pack.length - 20)); - const actual = sha1hex(pack.subarray(0, pack.length - 20)); - if (trailer !== actual) throw new Error("pack checksum mismatch"); - - const entries: RawEntry[] = []; - let pos = 12; - for (let i = 0; i < count; i++) { - const offset = pos; - let byte = pack[pos++]; - const type = (byte >> 4) & 7; - while (byte & 0x80) { - byte = pack[pos++]; - } - const entry: RawEntry = { offset, type, data: new Uint8Array(0) }; - if (type === OFS_DELTA) { - byte = pack[pos++]; - let off = byte & 0x7f; - while (byte & 0x80) { - byte = pack[pos++]; - off = (off + 1) * 128 + (byte & 0x7f); - } - entry.baseOffset = offset - off; - } else if (type === REF_DELTA) { - entry.baseOid = toHex(pack.subarray(pos, pos + 20)); - pos += 20; - } else if (!NUM_TYPE[type]) { - throw new Error(`bad object type ${type} at ${offset}`); - } - const { data, end } = inflateEntry(pack, pos); - entry.data = data; - pos = end; - if (type !== OFS_DELTA && type !== REF_DELTA) { - entry.resolved = { type: NUM_TYPE[type], data }; - } - entries.push(entry); - } - - // resolve deltas; bases can be other pack entries or (thin pack) store objects - const byOffset = new Map<number, RawEntry>(); - for (const e of entries) byOffset.set(e.offset, e); - const oidOf = new Map<RawEntry, string>(); - const byOid = new Map<string, RawEntry>(); - - const hashEntry = (e: RawEntry) => { - if (!e.resolved || oidOf.has(e)) return; - const oid = hashObject(e.resolved.type, e.resolved.data); - oidOf.set(e, oid); - byOid.set(oid, e); - }; - for (const e of entries) hashEntry(e); - - let unresolved = entries.filter((e) => !e.resolved); - while (unresolved.length) { - let progress = false; - const next: RawEntry[] = []; - for (const e of unresolved) { - let base: { type: ObjType; data: Uint8Array } | null = null; - if (e.baseOffset !== undefined) { - base = byOffset.get(e.baseOffset)?.resolved ?? null; - } else if (e.baseOid) { - base = byOid.get(e.baseOid)?.resolved ?? getBase(e.baseOid); - } - if (base) { - e.resolved = { type: base.type, data: applyDelta(base.data, e.data) }; - hashEntry(e); - progress = true; - } else { - next.push(e); - } - } - if (!progress) throw new Error(`cannot resolve ${next.length} delta object(s): missing base`); - unresolved = next; - } - - return entries.map((e) => ({ oid: oidOf.get(e)!, type: e.resolved!.type, data: e.resolved!.data })); -} - const REF_DELTA_NUM = 7; function encodeTypeSizeNum(typeNum: number, size: number): Uint8Array { @@ -222,13 +122,3 @@ this.emit(this.sha.digest()); // trailer is not part of the hashed content } } - -/** Convenience: build a whole pack in memory (used for small internal packs). */ -export function buildPack(objects: { type: ObjType; data: Uint8Array }[]): Uint8Array { - const parts: Uint8Array[] = []; - const w = new PackWriter((c) => parts.push(c)); - w.header(objects.length); - for (const o of objects) w.object(o.type, o.data); - w.finish(); - return concat(parts); -} diff --git a/src/git/packstore.ts b/src/git/packstore.ts @@ -1,10 +1,10 @@ import pako from "pako"; import { td, concat, toHex, Sha1 } from "./util"; +import { Sha1Dc, Sha1CollisionError } from "./sha1"; import { ObjType, NUM_TYPE, objectHeader } from "./objects"; import { applyDelta } from "./pack"; export const PACK_CHUNK = 1024 * 1024; -const SUBTLE_THRESHOLD = 256 * 1024; /** real Workers isolates have a hard 128MB total; self-hosted celld nodes run multi-GB heaps */ const TIGHT_MEMORY = typeof caches !== "undefined"; const MAX_BUFFERED_ENTRY = (TIGHT_MEMORY ? 8 : 32) * 1024 * 1024; @@ -12,6 +12,20 @@ const CACHE_ENTRY_LIMIT = (TIGHT_MEMORY ? 2 : 4) * 1024 * 1024; /** recent entry offsets kept in memory for ofs-delta base resolution */ const OFFSET_WINDOW = 150_000; +/** + * Decoded pack chunks kept hot for readRaw. The clone loop reads one object + * per call in walk order, so without this every object re-SELECTs and + * re-decodes a whole 1MB row. A handful of 1MB chunks covers any delta chain + * and the locality of the walk. + */ +const RAW_CHUNK_CACHE = TIGHT_MEMORY ? 4 : 8; +/** + * Hard cap on delta base-chain length when resolving an object. git's default + * pack depth is 50; 100 leaves generous headroom for legitimately deep packs + * while a crafted deeper (or cyclic) chain is rejected instead of overflowing + * the stack or looping forever. + */ +const MAX_DELTA_DEPTH = 100; export interface ObjRec { type: ObjType; @@ -62,15 +76,28 @@ } // ingest is strictly sequential, so scratch hashers serve every object -const scratchSha = new Sha1(); -const streamSha = new Sha1(); +const scratchSha = new Sha1Dc(); +const streamSha = new Sha1Dc(); +/** + * Finish an object hash, refusing anything carrying a SHA-1 collision-attack + * block. This is git's post-SHAttered rule: the id is still plain SHA-1, but + * an object built to collide never enters the database. + */ +function finishOid(h: Sha1Dc): string { + const oid = toHex(h.digest()); + if (h.collision) throw new Sha1CollisionError(oid); + return oid; +} + +/** + * Every ingested object goes through the collision-detecting hash, so there + * is no fast path here: crypto.subtle is ~10x quicker on large blobs but + * cannot screen for near-collision blocks, and an attacker would only have to + * pad past the threshold to skip the check. + */ async function hashObjectAsync(type: ObjType, data: Uint8Array): Promise<string> { - if (data.length >= SUBTLE_THRESHOLD) { - const full = concat([objectHeader(type, data.length), data]); - return toHex(new Uint8Array(await crypto.subtle.digest("SHA-1", full as BufferSource))); - } - return toHex(scratchSha.reset().update(objectHeader(type, data.length)).update(data).digest()); + return finishOid(scratchSha.reset().update(objectHeader(type, data.length)).update(data)); } /** Buffered sequential reader over a pack stored as chunk rows. */ @@ -115,6 +142,9 @@ * delta compression. This is what lets Linux-sized repos fit and stream. */ export class PackStore { + /** LRU of decoded `pack_data` rows, keyed "packId:seq". */ + private chunks = new Map<string, Uint8Array>(); + constructor(private sql: SqlStorage, private extern: ExternalResolver) { this.init(); } @@ -162,6 +192,7 @@ for (const t of ["pack_meta", "pack_data", "pack_objects", "pack_pending"]) { this.sql.exec(`DROP TABLE IF EXISTS ${t}`); } + this.chunks.clear(); // pack ids restart from 1: cached rows would be stale } /** Drop all packs and start empty (small-repo gc migrates objects out first). */ @@ -174,6 +205,21 @@ return this.sql.exec<{ n: number }>("SELECT COUNT(*) AS n FROM pack_objects").one().n; } + /** Total stored bytes across all indexed packs (0 when pack-empty). */ + totalPackBytes(): number { + return this.sql.exec<{ n: number }>("SELECT COALESCE(SUM(size), 0) AS n FROM pack_meta").one().n; + } + + /** + * Bytes of pack storage the raw-chunk LRU keeps hot. A pack no larger than + * this fits entirely in cache, so readRaw never re-decodes a chunk no matter + * what order objects are emitted in — sorting for read locality is pure + * overhead below this threshold. + */ + rawCacheWindow(): number { + return RAW_CHUNK_CACHE * PACK_CHUNK; + } + lookup(oid: string): PackedEntry | null { const rows = this.sql .exec<{ @@ -199,6 +245,14 @@ return rows[0] ?? null; } + /** Distinct delta-base oids: a second reference edge gc must not sever. */ + baseOids(): string[] { + return this.sql + .exec<{ base_oid: string }>("SELECT DISTINCT base_oid FROM pack_objects WHERE base_oid IS NOT NULL") + .toArray() + .map((r) => r.base_oid); + } + findOidPrefix(prefix: string): string[] { return this.sql .exec<{ oid: string }>("SELECT oid FROM pack_objects WHERE oid LIKE ? LIMIT 2", prefix + "%") @@ -216,15 +270,7 @@ const last = Math.floor((off + len - 1) / PACK_CHUNK); const out = new Uint8Array(len); for (let seq = first; seq <= last; seq++) { - const rows = this.sql - .exec<{ data: ArrayBuffer }>( - "SELECT data FROM pack_data WHERE pack_id = ? AND seq = ?", - packId, - seq - ) - .toArray(); - if (!rows.length) continue; - const chunk = new Uint8Array(rows[0].data); + const chunk = this.chunk(packId, seq); const chunkStart = seq * PACK_CHUNK; const from = Math.max(off, chunkStart); const to = Math.min(off + len, chunkStart + chunk.length); @@ -233,22 +279,87 @@ return out; } - /** Inflate + delta-resolve an object out of pack storage. */ + /** + * One decoded `pack_data` row, through the LRU. Rows are immutable once + * written — a pack chunk is inserted exactly once and only ever removed + * wholesale (wipe/reset, or the orphan sweep at the top of ingest, both of + * which clear the cache), so a hit can never be stale. + */ + private chunk(packId: number, seq: number): Uint8Array { + const key = `${packId}:${seq}`; + const hit = this.chunks.get(key); + if (hit) { + this.chunks.delete(key); // reinsert: most-recently-used goes last + this.chunks.set(key, hit); + return hit; + } + const rows = this.sql + .exec<{ data: ArrayBuffer }>( + "SELECT data FROM pack_data WHERE pack_id = ? AND seq = ?", + packId, + seq + ) + .toArray(); + if (!rows.length) throw new Error(`pack ${packId}: missing chunk ${seq}`); + const chunk = new Uint8Array(rows[0].data); + this.chunks.set(key, chunk); + while (this.chunks.size > RAW_CHUNK_CACHE) { + this.chunks.delete(this.chunks.keys().next().value as string); + } + return chunk; + } + + /** + * Inflate + delta-resolve an object out of pack storage. A crafted pack can + * chain deltas arbitrarily deep or in a cycle, so the base chain is walked + * ITERATIVELY (not recursively) through cheap index lookups first — bounding + * the chain length and detecting cycles before a single delta is inflated — + * then applied from the base upward, holding at most two inflated buffers at + * a time. This caps both stack depth and peak memory regardless of input. + */ getObject(oid: string, cache: ObjCache): ObjRec | null { const cached = cache.get(oid); if (cached) return cached; - const entry = this.lookup(oid); - if (!entry) return null; - const raw = pako.inflate(this.readRaw(entry.packId, entry.dataOff, entry.dataLen)); - let obj: ObjRec; - if (entry.baseOid) { - const base = this.getObject(entry.baseOid, cache) ?? this.extern(entry.baseOid); - if (!base) throw new Error(`missing delta base ${entry.baseOid} for ${oid}`); - obj = { type: base.type, data: applyDelta(base.data, raw) }; - } else { - obj = { type: entry.type, data: raw }; + const first = this.lookup(oid); + if (!first) return null; + + // walk to the base via lookups only: collect the delta entries, resolve + // where the chain bottoms out (a full entry, a cached object, or a thin + // base from another pack / loose storage). + const chain: PackedEntry[] = []; + const seen = new Set<string>(); + let base: ObjRec | null = null; + let cur: PackedEntry = first; + for (;;) { + if (seen.has(cur.oid)) throw new Error(`cyclic delta chain at ${cur.oid}`); + seen.add(cur.oid); + if (!cur.baseOid) { + base = { type: cur.type, data: pako.inflate(this.readRaw(cur.packId, cur.dataOff, cur.dataLen)) }; + break; + } + if (chain.length >= MAX_DELTA_DEPTH) throw new Error(`delta chain exceeds depth ${MAX_DELTA_DEPTH} at ${oid}`); + chain.push(cur); + const cachedBase = cache.get(cur.baseOid); + if (cachedBase) { base = cachedBase; break; } + const next = this.lookup(cur.baseOid); + if (!next) { + const ext = this.extern(cur.baseOid); + if (!ext) throw new Error(`missing delta base ${cur.baseOid} for ${cur.oid}`); + base = ext; + break; + } + cur = next; } - cache.put(oid, obj); + + // apply deltas from the base upward (chain is target..base order) + let obj = base; + for (let i = chain.length - 1; i >= 0; i--) { + const d = chain[i]; + const delta = pako.inflate(this.readRaw(d.packId, d.dataOff, d.dataLen)); + obj = { type: base.type, data: applyDelta(obj.data, delta) }; + cache.put(d.oid, obj); + } + if (!chain.length) cache.put(oid, obj); return obj; } @@ -283,6 +394,9 @@ this.sql.exec(`DELETE FROM ${t} WHERE pack_id = ?`, o.pack_id); } } + // the reclaimed pack id is about to be handed out again: drop anything the + // chunk LRU still holds for it (and for anything else — it is only a cache) + if (orphans.length) this.chunks.clear(); const packId = (this.sql.exec<{ m: number | null }>("SELECT MAX(pack_id) AS m FROM pack_meta").one().m ?? 0) + 1; const started = Date.now(); @@ -472,7 +586,7 @@ content = pieces.length === 1 ? pieces[0] : concat(pieces); oid = await hashObjectAsync(objType!, content); } else { - oid = toHex(streamSha.digest()); + oid = finishOid(streamSha); } insertObject([oid, packId, offset, dataOff, dataLen, objType!, entrySize, entrySize, null]); winPut(offset, oid); @@ -516,6 +630,9 @@ if (i % 100000 === 99999) say(`Indexing objects: ${i + 1}/${count}\n`); } } + // the header count must account for every byte up to the 20-byte trailer; + // otherwise objects were silently dropped (or junk trails the pack) + if (r.pos !== total - 20) throw new Error("pack has trailing data or bad object count"); await opts.flush?.(); say(`Indexed ${count} objects (${pendingCount} deferred)\n`); diff --git a/src/git/pktline.ts b/src/git/pktline.ts @@ -20,10 +20,31 @@ pos = 0; constructor(private buf: Uint8Array) {} + /** + * Strict 4-byte hex length. `parseInt` is too lenient for attacker-framed + * input — it accepts leading whitespace and stops at the first non-digit, so + * " 12" or "001g" would parse to a bogus length and desync the stream. Every + * one of the four bytes must be a hex digit or the packet is rejected. + */ + private len4(): number { + let n = 0; + for (let i = 0; i < 4; i++) { + const c = this.buf[this.pos + i]; + let d: number; + if (c >= 0x30 && c <= 0x39) d = c - 0x30; + else if (c >= 0x61 && c <= 0x66) d = c - 0x57; + else if (c >= 0x41 && c <= 0x46) d = c - 0x37; + else throw new Error("bad pkt-line length"); + n = (n << 4) | d; + } + return n; + } + read(): Pkt | null { if (this.pos + 4 > this.buf.length) return null; - const len = parseInt(td.decode(this.buf.subarray(this.pos, this.pos + 4)), 16); - if (Number.isNaN(len)) throw new Error("bad pkt-line length"); + const len = this.len4(); + // 0000 flush and 0001 delim are the only special lengths; 0002/0003 are + // undefined and rejected, as is any declared length that overruns the buffer. if (len === 0) { this.pos += 4; return { kind: "flush", raw: new Uint8Array(0), text: "" }; diff --git a/src/git/protocol.ts b/src/git/protocol.ts @@ -3,12 +3,20 @@ import { GitStore } from "./store"; import { PackWriter } from "./pack"; import { OidSet } from "./oidset"; -import { parseCommit, parseTag, parseTree, isGitlinkMode, TYPE_NUM } from "./objects"; +import { parseCommit, parseTag, parseTree, isGitlinkMode, isTreeMode, TYPE_NUM, NUM_TYPE } from "./objects"; const AGENT = "agent=dgit/0.3"; const INFINITE_DEPTH = 0x7fffffff; const SIDEBAND_CHUNK = 32 * 1024; const WALK_YIELD = 5000; +/** + * Committer-date skew tolerated before the uninteresting (have) commit walk + * stops descending. Cutting the walk short can only ever shrink the "client + * already has it" set, i.e. send a few redundant objects — never omit one — + * so a day of slack is a safe way to keep an incremental fetch off the full + * commit graph. + */ +const WALK_DATE_SLOP = 24 * 60 * 60; export type Service = "git-upload-pack" | "git-receive-pack"; @@ -16,7 +24,14 @@ export function advertisement(store: GitStore, service: Service): Uint8Array { const caps = service === "git-upload-pack" - ? ["shallow", "side-band-64k", `symref=HEAD:${store.head()}`, AGENT].join(" ") + ? [ + "multi_ack_detailed", + "no-done", + "shallow", + "side-band-64k", + `symref=HEAD:${store.head()}`, + AGENT, + ].join(" ") : ["report-status", "delete-refs", "ofs-delta", "side-band-64k", AGENT].join(" "); const lines: Uint8Array[] = [pkt(`# service=${service}\n`), FLUSH]; @@ -38,6 +53,48 @@ return concat(lines); } +/** + * The oids a fetch is allowed to `want`: exactly what the advertisement above + * exposes — HEAD, every current ref tip, and the peeled commit target of each + * annotated tag (git advertises those as `<tag>^{}`). + * + * This is git's default (`uploadpack.allowAnySHA1InWant=false`): serving any + * oid the database merely *has* leaks objects that are no longer reachable + * from any ref — a commit orphaned by a force-push stays readable to anyone + * who learned its sha. Membership in a set of ref tips, so a normal fetch + * (which only ever wants advertised tips) pays a few hash lookups, not a walk. + */ +export function advertisedOids(store: GitStore): Set<string> { + const oids = new Set<string>(); + const head = store.resolveHead(); + if (head) oids.add(head); + for (const r of store.refs()) { + oids.add(r.target); + // peel annotated tags; lightweight tags already point at the commit + if (!r.name.startsWith("refs/tags/")) continue; + if (store.typeAndSize(r.target)?.type !== "tag") continue; + const peeled = peelToCommitOid(store, r.target); + if (peeled) oids.add(peeled); + } + return oids; +} + +/** + * True when `wants` is exactly the set of current ref tips — a true full clone, + * not a single-branch or partial fetch. Only then does the whole-repo reachable + * cache describe precisely what the client asked for; a subset of the tips would + * pull unrelated branches' objects in and leave the client with danglers. + */ +function wantsAreAllTips(store: GitStore, wants: string[]): boolean { + const tips = new Set<string>(); + for (const r of store.refs()) tips.add(r.target); + if (tips.size === 0) return false; + const wantSet = new Set(wants); + if (wantSet.size !== tips.size) return false; + for (const t of tips) if (!wantSet.has(t)) return false; + return true; +} + export function sidebandFrames(band: number, payload: Uint8Array): Uint8Array[] { const frames: Uint8Array[] = []; for (let off = 0; off < payload.length; off += SIDEBAND_CHUNK) { @@ -129,7 +186,10 @@ if (obj?.type !== "commit") continue; const parents = parseCommit(obj.data).parents; if (d >= depth) { - if (parents.length) boundary.add(oid); + // every commit reached at the depth limit is a graft point, root commits + // included — git's get_shallow_commits() marks them unconditionally, and + // the client compares the advertised list against its own .git/shallow + boundary.add(oid); continue; } boundary.delete(oid); // reachable within depth via this (shorter) path @@ -144,26 +204,89 @@ } /** - * Everything the client already has: closure of its haves, cut at its - * shallow boundaries (a shallow client does NOT have the parents of its - * shallow commits, nor their trees). + * Commits reachable from the wants without passing through one of the + * client's haves — the "interesting" side of the walk — plus the oldest + * committer date among them. Commit objects only; no trees are touched. + * + * The date is what bounds the uninteresting side: nothing older than the + * oldest thing the client is asking for can be needed as a boundary, so the + * have walk may stop there instead of marching down the whole commit graph. */ +async function interestingCommits( + store: GitStore, + wants: string[], + haveCommits: Set<string>, + yieldMaybe: () => Promise<void> +): Promise<{ set: OidSet; minTime: number }> { + const set = new OidSet(4096); + const stack: string[] = []; + for (const w of wants) { + const c = peelToCommitOid(store, w); + if (c && !haveCommits.has(c) && set.addHex(c)) stack.push(c); + } + let minTime = Infinity; + while (stack.length) { + const oid = stack.pop()!; + const obj = store.get(oid); + if (obj?.type !== "commit") continue; + await yieldMaybe(); + const c = parseCommit(obj.data); + if (c.committer.time < minTime) minTime = c.committer.time; + for (const p of c.parents) { + if (haveCommits.has(p)) continue; // the client's side of the boundary + if (store.has(p) && set.addHex(p)) stack.push(p); + } + } + return { set, minTime }; +} + +/** + * Objects the client demonstrably already has, marked the way git marks + * UNINTERESTING: commits reachable from the haves (cut at the client's + * shallow boundaries), plus the *tree closure of the boundary commits only* + * — the client's own tips and the uninteresting parents of the commits it is + * asking for. Boundary tree descent halts the moment it reaches an object + * already known present, so shared subtrees are walked once, and history the + * fetch never touches is never inflated at all. + * + * The set is deliberately allowed to be an under-approximation: leaving an + * object out only means re-sending something the client had (a bigger pack), + * while putting one in wrongly would produce a broken pack. Every bound here + * therefore errs towards sending. + */ async function excludedObjects( store: GitStore, haves: string[], - clientShallows: string[] + clientShallows: string[], + wants: string[] ): Promise<OidSet> { - const shallowStops = new Set(clientShallows); - const excluded = new OidSet(haves.length * 64); - const commitStack: string[] = []; + const excluded = new OidSet(Math.max(haves.length * 64, 4096)); + if (!haves.length) return excluded; // full clone: nothing is excluded + let ops = 0; const yieldMaybe = async () => { if (++ops % WALK_YIELD === 0) await new Promise((r) => setTimeout(r, 0)); }; + + // the client's tips, peeled through any annotated tags + const haveCommits = new Set<string>(); for (const h of haves) { + if (!store.has(h)) continue; + const c = peelToCommitOid(store, h); + if (c) haveCommits.add(c); + } + + const { set: interesting, minTime } = await interestingCommits(store, wants, haveCommits, yieldMaybe); + // minTime stays Infinity when the client wants nothing new: then no ancestor + // of a have can be a boundary and the walk stops at the haves themselves. + const cutoff = minTime === Infinity ? Infinity : minTime - WALK_DATE_SLOP; + + // uninteresting commit walk — commits and tags only, no trees + const shallowStops = new Set(clientShallows); + const commitStack: string[] = []; + for (const h of haves) { if (store.has(h) && excluded.addHex(h)) commitStack.push(h); } - const trees: string[] = []; while (commitStack.length) { const oid = commitStack.pop()!; const obj = store.get(oid); @@ -176,12 +299,36 @@ } if (obj.type !== "commit") continue; const c = parseCommit(obj.data); - if (excluded.addHex(c.tree)) trees.push(c.tree); if (shallowStops.has(oid)) continue; // client's history stops here + if (c.committer.time < cutoff) continue; // older than anything wanted for (const p of c.parents) { if (store.has(p) && excluded.addHex(p)) commitStack.push(p); } } + + // boundary commits whose trees are worth marking: the client's own tips + // (it has those trees in full) and the uninteresting parents of interesting + // commits (git's mark_edges_uninteresting). + const boundary: string[] = []; + for (const c of haveCommits) boundary.push(c); + for (let i = 0; i < interesting.size; i++) { + const oid = interesting.atHex(i); + if (excluded.hasHex(oid)) continue; // contested: reachable from a have too + const obj = store.get(oid); + if (obj?.type !== "commit") continue; + await yieldMaybe(); + for (const p of parseCommit(obj.data).parents) { + if (excluded.hasHex(p)) boundary.push(p); + } + } + + const trees: string[] = []; + for (const oid of boundary) { + const obj = store.get(oid); + if (obj?.type !== "commit") continue; + const tree = parseCommit(obj.data).tree; + if (tree && excluded.addHex(tree)) trees.push(tree); + } while (trees.length) { const oid = trees.pop()!; const obj = store.get(oid); @@ -189,7 +336,8 @@ await yieldMaybe(); for (const e of parseTree(obj.data)) { if (isGitlinkMode(e.mode)) continue; - if (excluded.addHex(e.oid) && store.typeAndSize(e.oid)?.type === "tree") trees.push(e.oid); + // already present => the whole subtree below it is too; stop descending + if (excluded.addHex(e.oid) && isTreeMode(e.mode)) trees.push(e.oid); } } return excluded; @@ -197,37 +345,44 @@ /** * Objects to pack: closure of wants, minus excluded, cut at commitLimit. - * An excluded commit is not re-sent, but the walk still descends through - * its parents: with a shallow client, commits BELOW its boundary are not - * excluded and must be reachable even when the walk enters via commits the - * client already has (deepen and unshallow fetches). Blobs are added by - * type lookup alone — their content is never inflated during the walk. + * Returned as one OidSet whose tagged sub-set is the pack contents — the + * untagged remainder is the walk's visited bookkeeping, which is a strict + * superset, so one set carries both roles at half the per-object cost. + * + * `descendThroughExcluded` is set only when the client is deepening: with a + * shallow client, commits BELOW its boundary are not excluded and must be + * reachable even when the walk enters via commits it already has (deepen and + * unshallow fetches). On an ordinary fetch the walk must stop dead at an + * excluded commit — descending anyway re-sends the history the client + * deliberately does not have, and strands it below its own shallow graft. + * Blobs are added by type lookup alone — never inflated during the walk. */ async function collectPackOids( store: GitStore, wants: string[], excluded: OidSet, - commitLimit: Set<string> | null + commitLimit: Set<string> | null, + descendThroughExcluded: boolean, + expected: number ): Promise<OidSet> { - const send = new OidSet(4096); - const visited = new OidSet(4096); + const walk = new OidSet(expected); const stack = [...wants]; let ops = 0; while (stack.length) { const oid = stack.pop()!; - if (!visited.addHex(oid)) continue; + if (!walk.addHex(oid)) continue; if (++ops % WALK_YIELD === 0) await new Promise((r) => setTimeout(r, 0)); const meta = store.typeAndSize(oid); if (!meta) throw new Error(`missing object ${oid}`); if (meta.type === "commit" && commitLimit && !commitLimit.has(oid)) continue; if (excluded.hasHex(oid)) { - if (meta.type === "commit") { + if (descendThroughExcluded && meta.type === "commit") { const full = store.get(oid)!; stack.push(...parseCommit(full.data).parents); } continue; } - send.addHex(oid); + walk.markHex(oid); if (meta.type === "blob") continue; // leaf: membership only const full = store.get(oid)!; if (full.type === "commit") { @@ -242,7 +397,7 @@ } } } - return send; + return walk; } /** @@ -251,7 +406,11 @@ * Stored pack entries are copied verbatim (deltas preserved) whenever their * base ships in the same pack; everything else is inflated and re-deflated. */ -export async function uploadPack(store: GitStore, body: Uint8Array): Promise<Response> { +export async function uploadPack( + store: GitStore, + body: Uint8Array, + release: () => void = () => {} +): Promise<Response> { const headers = { "content-type": "application/x-git-upload-pack-result", "cache-control": "no-cache", @@ -258,10 +417,14 @@ }; const req = parseUploadRequest(body); if (!req.wants.length || req.wants.some((w) => !isOid(w))) { + release(); return new Response(pkt("ERR no valid wants\n") as unknown as BodyInit, { headers }); } + // a want must be an object we currently advertise, not merely one we store + const allowed = advertisedOids(store); for (const w of req.wants) { - if (!store.has(w)) { + if (!allowed.has(w) || !store.has(w)) { + release(); return new Response(pkt(`ERR upload-pack: not our ref ${w}\n`) as unknown as BodyInit, { headers }); } } @@ -290,21 +453,174 @@ preamble.push(FLUSH); } - // single-ack negotiation: ACK the first common object, else NAK. A round - // with neither haves nor done (shallow discovery) gets no ack line at all — - // a stray NAK would desync the client's stateless response stream. - const firstCommon = req.haves.find((h) => isOid(h) && store.has(h)); - if (req.done || req.haves.length) { - preamble.push(pkt(firstCommon ? `ACK ${firstCommon}\n` : "NAK\n")); + // Negotiation. A round with neither haves nor done (shallow discovery) gets + // no ack line at all — a stray NAK would desync the client's stateless + // response stream. + // + // Single-ack cannot carry a stateless fetch on its own: `ACK <oid>` means + // "negotiation over, pack follows", so a client that receives one mid-round + // stops negotiating and reads the response as a pack — but git only batches + // 16 haves per round, and the follow-up request it sends carries no haves at + // all. The pack therefore has to be produced by the same request that + // carried the haves, which is exactly what multi_ack_detailed's `ready` plus + // `no-done` is for. This mirrors upload-pack's wire shape line for line; + // clients that negotiate neither capability keep the single-ack path. + const common = req.haves.filter((h) => isOid(h) && store.has(h)); + const lastCommon = common.length ? common[common.length - 1] : null; + let sendPack = req.done; + if (!req.caps.has("multi_ack_detailed")) { + if (req.done || req.haves.length) { + preamble.push(pkt(common.length ? `ACK ${common[0]}\n` : "NAK\n")); + } + } else if (req.done) { + for (const c of common) preamble.push(pkt(`ACK ${c} common\n`)); + preamble.push(pkt(lastCommon ? `ACK ${lastCommon}\n` : "NAK\n")); + } else if (req.haves.length) { + for (const c of common) preamble.push(pkt(`ACK ${c} common\n`)); + if (lastCommon && req.caps.has("no-done")) { + // enough common ground to build the pack, and the client agreed it can + // be answered without a `done` round trip: reply with it right here, + // while the haves are still in hand + preamble.push(pkt(`ACK ${lastCommon} ready\n`)); + preamble.push(pkt("NAK\n")); + preamble.push(pkt(`ACK ${lastCommon}\n`)); + sendPack = true; + } else { + preamble.push(pkt("NAK\n")); // keep negotiating + } } - if (!req.done) { + if (!sendPack) { // negotiation round only — client will POST again + release(); return new Response(concat(preamble) as unknown as BodyInit, { headers }); } - const excluded = await excludedObjects(store, req.haves, req.clientShallows); - const send = await collectPackOids(store, req.wants, excluded, commitLimit); + // Full-clone fast path: no haves, not shallow/deepen, and the wants are + // exactly the current ref tips. The reachable object set is then identical to + // what the graph walk would rediscover, so serve it from the versioned cache + // and skip the walk (and every commit/tree inflation it drives) entirely. A + // missing or stale cache (version mismatch, or any failure) falls straight + // through to the walk, which then repopulates it — so a wrong pack is never + // served, and existing repos become fast on their next full clone. + const fullClone = !req.haves.length && req.deepen === 0 && wantsAreAllTips(store, req.wants); + let send = fullClone ? store.loadReachable() : null; + if (!send) { + const excluded = await excludedObjects(store, req.haves, req.clientShallows, req.wants); + // a plain clone enumerates the whole database: size the walk's set up front + // so it never doubles (a doubling keeps the old and new arrays live at once) + const expected = + !req.haves.length && !commitLimit && !req.deepen ? Math.max(store.objectCount(), 4096) : 4096; + send = await collectPackOids( + store, + req.wants, + excluded, + commitLimit, + req.deepen > 0, // descendThroughExcluded: only while deepening/unshallowing + expected + ); + if (fullClone) { + try { + await store.saveReachable(send); + } catch { + // the cache is best-effort: a failed persist just means the next full + // clone walks again, never a wrong pack + } + } + } + // Emit objects in pack storage order — (packId, offset) — rather than the + // DFS walk order they were collected in. Reused entries are copied straight + // out of stored packs via readRaw, and the raw-chunk LRU only pays off when + // consecutive reads stay inside the same 1MB pack chunk; walk order jumps + // all over the pack, so it rarely does. Marching sequentially through each + // pack lifts that hit rate to ~100% and cuts per-object decode cost. + // + // Order-safe: the SET of emitted entries and every entry's bytes are decided + // by set membership alone (isMarkedHex on the delta base), never by position, + // so only the ordering changes. Delta entries are re-addressed as REF-deltas + // (see PackWriter.rawDelta) and git's index-pack resolves ref-deltas in any + // order via a second pass; no ofs-deltas are ever emitted. Loose objects + // (no pack entry) sort last, ordered by oid, for a stable total order. + // + // Only worth it once the packs outgrow the raw-chunk LRU: below that every + // chunk stays cached regardless of order, so the sort would be pure overhead. + // The arrays stay null then and emission falls back to walk order with a + // per-object lookup, exactly as before. + // + // The lookups done to build the sort key are *retained* — in LEAN parallel + // typed arrays, not an array of PackedEntry objects — and reused by the + // stream, so sorting adds an in-memory sort but not a second round of SQL + // lookups. Storing the reused fields as five typed arrays (~28 B/obj) rather + // than live objects (~204 B/obj) is what keeps a one-shot full clone inside + // the 128MB isolate at multi-million-object scale. + // + // ePack Int32 pack id (1-based), or LOOSE_PACK for objects with no + // pack entry (loose / unindexed) — those sort last, by oid. + // eDataOff Float64 start of the stored (compressed) region. Offsets on a + // pack past 4GB exceed 2^32, so this must NOT be a Uint32. The + // (packId, dataOff) pair is also the sort key: dataOff rises + // monotonically with entry offset within a pack, so ordering by + // it reproduces pack storage order exactly. + // eDataLen Uint32 length of that region (a single entry never reaches + // the 4GB a Uint32 caps, and readRaw could not allocate one). + // eSize Float64 uncompressed entry size for the pack object header. + // A blob copied verbatim on a big-heap celld node can exceed 4GB + // uncompressed, so this is not a Uint32 either. + // eCode Int32 how to emit the object, folding type and delta base + // into one slot (they are mutually exclusive per entry): + // >= 0 ref-delta whose base ships in this pack; the value is + // the base's insertion index in `send` (base oid = + // send.atHex(code)). + // <= -3 full entry copied verbatim; NUM_TYPE[-2 - code] is its + // type (commit=-3, tree=-4, blob=-5, tag=-6). + // -1 inflate via store.get: a loose object, or a delta whose + // base is not being sent (so it cannot be a ref-delta). + const total = send.markedSize; + let order: Int32Array | null = null; + let ePack: Int32Array | null = null; + let eDataOff: Float64Array | null = null; + let eDataLen: Uint32Array | null = null; + let eSize: Float64Array | null = null; + let eCode: Int32Array | null = null; + const LOOSE_PACK = 0x7fffffff; // sorts after every real (1-based) pack id + if (store.packs.totalPackBytes() > store.packs.rawCacheWindow()) { + ePack = new Int32Array(total); + eDataOff = new Float64Array(total); + eDataLen = new Uint32Array(total); + eSize = new Float64Array(total); + eCode = new Int32Array(total); + for (let i = 0; i < total; i++) { + const entry = store.packs.lookup(send.markedAtHex(i)); + if (!entry) { + ePack[i] = LOOSE_PACK; + eCode[i] = -1; + } else { + ePack[i] = entry.packId; + eDataOff[i] = entry.dataOff; + eDataLen[i] = entry.dataLen; + eSize[i] = entry.entrySize; + if (entry.baseOid) { + // ref-delta only when the base also ships here; else inflate it + eCode[i] = send.isMarkedHex(entry.baseOid) ? send.indexOfHex(entry.baseOid) : -1; + } else { + eCode[i] = -2 - TYPE_NUM[entry.type]; + } + } + if ((i & 8191) === 8191) await new Promise((r) => setTimeout(r, 0)); + } + order = new Int32Array(total); + for (let i = 0; i < total; i++) order[i] = i; + order.sort((a, b) => { + if (ePack![a] !== ePack![b]) return ePack![a] - ePack![b]; + if (ePack![a] === LOOSE_PACK) { + const oa = send.markedAtHex(a); + const ob = send.markedAtHex(b); + return oa < ob ? -1 : oa > ob ? 1 : 0; + } + return eDataOff![a] - eDataOff![b]; + }); + } + const sideband = req.caps.has("side-band-64k"); const noProgress = req.caps.has("no-progress"); @@ -328,7 +644,6 @@ let i = 0; let finished = false; - const total = send.size; const stream = new ReadableStream<Uint8Array>({ start: (ctrl) => { if (sideband && !noProgress) { @@ -341,13 +656,32 @@ pull: (ctrl) => { try { for (let n = 0; n < 64 && i < total; n++, i++) { - const oid = send.atHex(i); - const entry = store.packs.lookup(oid); - if (entry && !entry.baseOid) { - writer.rawFull(entry.type, entry.entrySize, store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen)); - } else if (entry && entry.baseOid && send.hasHex(entry.baseOid)) { - writer.rawDelta(entry.entrySize, entry.baseOid, store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen)); + const idx = order ? order[i] : i; + if (!eCode) { + // unsorted (small-pack) path: no sort pass ran, so do a fresh + // per-object lookup in walk order, exactly as before OPT 2. + const oid = send.markedAtHex(idx); + const entry = store.packs.lookup(oid); + if (entry && !entry.baseOid) { + writer.rawFull(entry.type, entry.entrySize, store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen)); + } else if (entry && entry.baseOid && send.isMarkedHex(entry.baseOid)) { + writer.rawDelta(entry.entrySize, entry.baseOid, store.packs.readRaw(entry.packId, entry.dataOff, entry.dataLen)); + } else { + const obj = store.get(oid); + if (!obj) throw new Error(`missing object ${oid}`); + writer.object(obj.type, obj.data); + } + continue; + } + // sorted path: everything needed to emit the object was captured in + // the lean typed arrays, so no SQL lookup happens here. + const code = eCode[idx]; + if (code <= -3) { + writer.rawFull(NUM_TYPE[-2 - code], eSize![idx], store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx])); + } else if (code >= 0) { + writer.rawDelta(eSize![idx], send.atHex(code), store.packs.readRaw(ePack![idx], eDataOff![idx], eDataLen![idx])); } else { + const oid = send.markedAtHex(idx); const obj = store.get(oid); if (!obj) throw new Error(`missing object ${oid}`); writer.object(obj.type, obj.data); @@ -363,13 +697,18 @@ } for (const c of pending) ctrl.enqueue(c); pending.length = 0; - if (finished) ctrl.close(); + if (finished) { + ctrl.close(); + release(); // stream fully drained: this upload no longer holds a slot + } } catch (err) { + release(); ctrl.error(err); } }, cancel: () => { - // client went away mid-transfer (e.g. negotiation round abort); nothing to clean up + // client went away mid-transfer (e.g. negotiation round abort) + release(); }, }); return new Response(stream, { headers }); @@ -427,6 +766,43 @@ return { commands, caps }; } +/** Characters git's check_ref_format rejects, plus HTML-dangerous ones. */ +const BAD_REF = /[\s~^:?*\[\\'"<>&`\x00-\x1f\x7f]/; + +/** + * Every object in the closure of `tip` is present, so setting a ref to it + * cannot brick the repo. A thin/broken pack that fails to close over its new + * ref leaves a referenced object absent; walking commit->tree->subtrees+blobs + * (and tag->target) with a presence probe at each node catches it before the + * ref moves. `known` is pre-seeded with the pre-existing ref tips, whose + * closure was validated at their own push, so descent stops there — the walk + * touches only objects this push introduced. + */ +function reachableComplete(store: GitStore, tip: string, known: OidSet): boolean { + const stack = [tip]; + while (stack.length) { + const oid = stack.pop()!; + if (!known.addHex(oid)) continue; + const meta = store.typeAndSize(oid); + if (!meta) return false; + if (meta.type === "blob") continue; + const obj = store.get(oid); + if (!obj) return false; + if (obj.type === "commit") { + const c = parseCommit(obj.data); + stack.push(c.tree, ...c.parents); + } else if (obj.type === "tag") { + const t = parseTag(obj.data); + if (t.object) stack.push(t.object); + } else if (obj.type === "tree") { + for (const e of parseTree(obj.data)) { + if (!isGitlinkMode(e.mode)) stack.push(e.oid); + } + } + } + return true; +} + /** Apply ref updates after the pack (if any) has been ingested. */ export function applyPushCommands( store: GitStore, @@ -436,12 +812,30 @@ const results: CommandResult[] = []; let changed = false; let needsGc = false; + // whether the reachable cache is valid for the pre-push refs: only then can + // it be extended in place (union with the connectivity walk) instead of + // rebuilt from scratch on the next full clone + const hadCache = store.getMeta("reachable-version") === store.reachableVersion(); + // pre-existing ref tips bound the connectivity walk: their closure is + // already validated, so a normal fast-forward only re-checks new objects + const known = new OidSet(4096); + for (const r of store.refs()) { + known.addHex(r.target); + const c = peelToCommitOid(store, r.target); + if (c) known.addHex(c); + } + const knownHead = store.resolveHead(); + if (knownHead) known.addHex(knownHead); for (const cmd of commands) { if (unpackError) { results.push({ ref: cmd.ref, ok: false, msg: "unpacker error" }); continue; } - if (!cmd.ref.startsWith("refs/") || cmd.ref.includes("..") || /[\s~^:?*\[\\]/.test(cmd.ref)) { + const bad = !cmd.ref.startsWith("refs/") || cmd.ref.includes("..") || BAD_REF.test(cmd.ref) + || cmd.ref.endsWith(".lock") || cmd.ref.includes("@{") || cmd.ref.includes("//") + || cmd.ref.endsWith("/") || cmd.ref.length > 255 + || cmd.ref.split("/").some((c) => c.startsWith(".")); + if (bad) { results.push({ ref: cmd.ref, ok: false, msg: "funny refname" }); continue; } @@ -454,7 +848,7 @@ store.delRef(cmd.ref); needsGc = true; } else { - if (!store.has(cmd.next)) { + if (!reachableComplete(store, cmd.next, known)) { results.push({ ref: cmd.ref, ok: false, msg: "missing necessary objects" }); continue; } @@ -476,6 +870,17 @@ branches[0]; if (preferred) store.setHead(preferred.name); } + + // maintain the full-clone reachable cache. `known` holds the connectivity + // walk: the pre-existing tips plus every object this push newly connected. On + // a pure fast-forward with a valid prior cache, union closes over exactly the + // new reachable set. Anything else (a force-push/delete that strands history, + // or no prior cache to extend) invalidates it, so the next full clone rebuilds + // by walking rather than risk serving objects that are no longer reachable. + if (changed) { + if (hadCache && !needsGc) store.extendReachable(known); + else store.invalidateReachable(); + } return { results, changed, needsGc }; } diff --git a/src/git/sha1.ts b/src/git/sha1.ts @@ -1,3 +1,5 @@ +import { SHA1_DVS, ubcCheck } from "./sha1dc-tables"; + /** * Incremental SHA-1. Workers' crypto.subtle is one-shot and async; pack * streaming needs a running digest and the object database wants sync @@ -114,3 +116,252 @@ export function sha1(data: Uint8Array): Uint8Array { return new Sha1().update(data).digest(); } + +/* + * SHA-1 collision detection (SHA-1DC) + * + * A port of Stevens & Shumow's sha1collisiondetection, the same defence + * git adopted after SHAttered. Every compressed block is screened by + * ubcCheck against 32 known disturbance vectors; a flagged DV is then + * confirmed by recompressing the block from the DV's test step with the + * perturbed message. If the reconstructed chaining value reproduces the + * real one, the block is half of a collision attack. + * + * Like git, safe-hash mangling stays off: the digest is always plain + * SHA-1, so object ids never move. Detection is reported out-of-band and + * the caller refuses the object. + */ + +const DVS = SHA1_DVS.map((dv) => ({ + testt: dv.testt, + maskb: dv.maskb, + dm: Int32Array.from(dv.dm), +})); + +const K0 = 0x5a827999 | 0; +const K1 = 0x6ed9eba1 | 0; +const K2 = 0x8f1bbcdc | 0; +const K3 = 0xca62c1d6 | 0; + +/** + * One step of the compression function, run over a rotating register file + * so a single loop can serve any step index. At step `i` the roles + * (a,b,c,d,e) live at v[(role - i) mod 5]. + */ +function stepForward(v: Int32Array, i: number, m: Int32Array): void { + const ia = (5 - (i % 5)) % 5; + const ib = (ia + 1) % 5; + const ic = (ia + 2) % 5; + const id = (ia + 3) % 5; + const ie = (ia + 4) % 5; + const a = v[ia], b = v[ib], c = v[ic], d = v[id], e = v[ie]; + let f: number, k: number; + if (i < 20) { f = d ^ (b & (c ^ d)); k = K0; } + else if (i < 40) { f = b ^ c ^ d; k = K1; } + else if (i < 60) { f = (b & c) | (d & (b ^ c)); k = K2; } + else { f = b ^ c ^ d; k = K3; } + v[ie] = (e + (((a << 5) | (a >>> 27)) + f + k + m[i])) | 0; + v[ib] = (b << 30) | (b >>> 2); +} + +/** Inverse of stepForward: undoes step `i`. */ +function stepBackward(v: Int32Array, i: number, m: Int32Array): void { + const ia = (5 - (i % 5)) % 5; + const ib = (ia + 1) % 5; + const ic = (ia + 2) % 5; + const id = (ia + 3) % 5; + const ie = (ia + 4) % 5; + const a = v[ia]; + const b = ((v[ib] >>> 30) | (v[ib] << 2)) | 0; + v[ib] = b; + const c = v[ic], d = v[id], e = v[ie]; + let f: number, k: number; + if (i < 20) { f = d ^ (b & (c ^ d)); k = K0; } + else if (i < 40) { f = b ^ c ^ d; k = K1; } + else if (i < 60) { f = (b & c) | (d & (b ^ c)); k = K2; } + else { f = b ^ c ^ d; k = K3; } + v[ie] = (e - (((a << 5) | (a >>> 27)) + f + k + m[i])) | 0; +} + +/** + * Reconstruct the chaining values a block would have had, had it been + * compressed with the perturbed message `me2`. Runs backwards from step + * `t` to recover the input chaining value, then forwards to step 79 for + * the output. `state` is the real block's register file entering step t. + */ +function recompress( + t: number, + ihvin: Int32Array, + ihvout: Int32Array, + me2: Int32Array, + state: Int32Array, + scratch: Int32Array +): void { + scratch.set(state); + for (let i = t - 1; i >= 0; i--) stepBackward(scratch, i, me2); + ihvin.set(scratch); + scratch.set(state); + for (let i = t; i < 80; i++) stepForward(scratch, i, me2); + for (let j = 0; j < 5; j++) ihvout[j] = (ihvin[j] + scratch[j]) | 0; +} + +/** + * Collision-detecting SHA-1. Drop-in for {@link Sha1}: `digest()` returns + * the identical bytes for every input. After digesting, {@link collision} + * reports whether any block looked like a collision-attack near-collision + * block. + */ +export class Sha1Dc { + private ihv = new Int32Array(5); + private block = new Uint8Array(64); + private blockLen = 0; + private bytes = 0; + /** expanded message of the block being compressed */ + private m1 = new Int32Array(80); + private m2 = new Int32Array(80); + /** register file entering steps 58 and 65 — the only DV test steps */ + private state58 = new Int32Array(5); + private state65 = new Int32Array(5); + private ihvin = new Int32Array(5); + private ihvout = new Int32Array(5); + private scratch = new Int32Array(5); + private found = false; + + constructor() { + this.reset(); + } + + /** True when a near-collision block was seen since the last reset. */ + get collision(): boolean { + return this.found; + } + + reset(): this { + this.ihv[0] = 0x67452301 | 0; + this.ihv[1] = 0xefcdab89 | 0; + this.ihv[2] = 0x98badcfe | 0; + this.ihv[3] = 0x10325476 | 0; + this.ihv[4] = 0xc3d2e1f0 | 0; + this.blockLen = 0; + this.bytes = 0; + this.found = false; + return this; + } + + update(data: Uint8Array): this { + this.bytes += data.length; + let off = 0; + if (this.blockLen > 0) { + const need = 64 - this.blockLen; + const take = Math.min(need, data.length); + this.block.set(data.subarray(0, take), this.blockLen); + this.blockLen += take; + off = take; + if (this.blockLen === 64) { + this.process(this.block, 0); + this.blockLen = 0; + } + } + while (off + 64 <= data.length) { + this.process(data, off); + off += 64; + } + if (off < data.length) { + this.block.set(data.subarray(off), 0); + this.blockLen = data.length - off; + } + return this; + } + + digest(): Uint8Array { + const bitLenHi = Math.floor((this.bytes * 8) / 0x100000000); + const bitLenLo = (this.bytes * 8) >>> 0; + const pad = new Uint8Array(((this.blockLen < 56 ? 56 : 120) - this.blockLen) + 8); + pad[0] = 0x80; + const dv = new DataView(pad.buffer); + dv.setUint32(pad.length - 8, bitLenHi); + dv.setUint32(pad.length - 4, bitLenLo); + this.update(pad); + const out = new Uint8Array(20); + const ov = new DataView(out.buffer); + for (let i = 0; i < 5; i++) ov.setInt32(i * 4, this.ihv[i]); + return out; + } + + /** Compress one block, retaining what the collision check needs. */ + private process(buf: Uint8Array, off: number): void { + const w = this.m1; + for (let i = 0; i < 16; i++) { + const j = off + i * 4; + w[i] = (buf[j] << 24) | (buf[j + 1] << 16) | (buf[j + 2] << 8) | buf[j + 3]; + } + for (let i = 16; i < 80; i++) { + const n = w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]; + w[i] = (n << 1) | (n >>> 31); + } + const ihv = this.ihv; + let a = ihv[0], b = ihv[1], c = ihv[2], d = ihv[3], e = ihv[4]; + for (let i = 0; i < 80; i++) { + // The unrolled reference names registers canonically while this loop + // shifts them, so a snapshot at step i must be rotated back by i mod 5. + if (i === 58) { + // 58 mod 5 == 3 + this.state58[0] = d; this.state58[1] = e; this.state58[2] = a; + this.state58[3] = b; this.state58[4] = c; + } else if (i === 65) { + // 65 mod 5 == 0 + this.state65[0] = a; this.state65[1] = b; this.state65[2] = c; + this.state65[3] = d; this.state65[4] = e; + } + let f: number, k: number; + if (i < 20) { f = (b & c) | (~b & d); k = K0; } + else if (i < 40) { f = b ^ c ^ d; k = K1; } + else if (i < 60) { f = (b & c) | (b & d) | (c & d); k = K2; } + else { f = b ^ c ^ d; k = K3; } + const t = (((a << 5) | (a >>> 27)) + f + e + k + w[i]) | 0; + e = d; + d = c; + c = (b << 30) | (b >>> 2); + b = a; + a = t; + } + ihv[0] = (ihv[0] + a) | 0; + ihv[1] = (ihv[1] + b) | 0; + ihv[2] = (ihv[2] + c) | 0; + ihv[3] = (ihv[3] + d) | 0; + ihv[4] = (ihv[4] + e) | 0; + + if (this.found) return; // already refused; skip the rest of the work + const mask = ubcCheck(w); + if (mask === 0) return; + const m2 = this.m2; + for (const dv of DVS) { + if ((mask & (1 << dv.maskb)) === 0) continue; + const dm = dv.dm; + for (let j = 0; j < 80; j++) m2[j] = w[j] ^ dm[j]; + recompress( + dv.testt, + this.ihvin, + this.ihvout, + m2, + dv.testt === 58 ? this.state58 : this.state65, + this.scratch + ); + const o = this.ihvout; + if (((o[0] ^ ihv[0]) | (o[1] ^ ihv[1]) | (o[2] ^ ihv[2]) | (o[3] ^ ihv[3]) | (o[4] ^ ihv[4])) === 0) { + this.found = true; + return; + } + } + } +} + +/** Thrown when a hashed object carries a SHA-1 collision-attack block. */ +export class Sha1CollisionError extends Error { + readonly oid: string; + constructor(oid: string) { + super(`object ${oid.slice(0, 12)} triggered SHA-1 collision detection; refused`); + this.name = "Sha1CollisionError"; + this.oid = oid; + } +} diff --git a/src/git/sha1dc-tables.ts b/src/git/sha1dc-tables.ts @@ -0,0 +1,322 @@ +// GENERATED — do not edit by hand. +// Transcribed from sha1collisiondetection/lib/ubc_check.c (Marc Stevens, Dan +// Shumow; MIT). Regenerate with scripts/gen-sha1dc-tables.mjs. +// +// C uint32 arithmetic maps onto JS int32 bit-for-bit: '>>' becomes '>>>' and +// wrapping subtraction is left to two's-complement, since every result feeds a +// bitwise operator. + +/** One disturbance vector: the step to recompress from, its bit in the UBC + * mask, and the expanded-message XOR difference. */ +export interface Sha1Dv { + readonly testt: number; + readonly maskb: number; + readonly dm: readonly number[]; +} + +/** The 32 disturbance vectors checked by SHA-1DC. */ +export const SHA1_DVS: readonly Sha1Dv[] = [ + { testt: 58, maskb: 0, dm: [0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6,0x8000004c,0x00000803,0x80000161,0x80000599] }, // I(43,0) + { testt: 58, maskb: 1, dm: [0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6,0x8000004c,0x00000803,0x80000161] }, // I(44,0) + { testt: 58, maskb: 2, dm: [0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6,0x8000004c,0x00000803] }, // I(45,0) + { testt: 58, maskb: 3, dm: [0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6,0x8000004c] }, // I(46,0) + { testt: 58, maskb: 4, dm: [0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060,0x00000590,0x00001020,0x0000039a,0x00000132] }, // I(46,2) + { testt: 58, maskb: 5, dm: [0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6] }, // I(47,0) + { testt: 58, maskb: 6, dm: [0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060,0x00000590,0x00001020,0x0000039a] }, // I(47,2) + { testt: 58, maskb: 7, dm: [0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408] }, // I(48,0) + { testt: 58, maskb: 8, dm: [0xe000002a,0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060,0x00000590,0x00001020] }, // I(48,2) + { testt: 58, maskb: 9, dm: [0x18000000,0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164] }, // I(49,0) + { testt: 58, maskb: 10, dm: [0x60000000,0xe000002a,0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060,0x00000590] }, // I(49,2) + { testt: 65, maskb: 11, dm: [0x0800000c,0x18000000,0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018] }, // I(50,0) + { testt: 65, maskb: 12, dm: [0x20000030,0x60000000,0xe000002a,0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060] }, // I(50,2) + { testt: 65, maskb: 13, dm: [0xe8000000,0x0800000c,0x18000000,0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202] }, // I(51,0) + { testt: 65, maskb: 14, dm: [0xa0000003,0x20000030,0x60000000,0xe000002a,0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a] }, // I(51,2) + { testt: 65, maskb: 15, dm: [0x04000010,0xe8000000,0x0800000c,0x18000000,0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012] }, // I(52,0) + { testt: 58, maskb: 16, dm: [0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d,0x8000041a,0x000002e4,0x80000054,0x00000967] }, // II(45,0) + { testt: 58, maskb: 17, dm: [0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d,0x8000041a,0x000002e4,0x80000054] }, // II(46,0) + { testt: 58, maskb: 18, dm: [0x90000070,0xb0000053,0x30000008,0x00000043,0xd0000072,0xb0000010,0xf0000062,0xc0000042,0x00000030,0xe0000042,0x20000060,0xe0000041,0x20000050,0xc0000041,0xe0000072,0xa0000003,0xc0000012,0x60000041,0xc0000032,0x20000001,0xc0000002,0xe0000042,0x60000042,0x80000002,0x00000000,0x00000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000001,0x00000060,0x80000003,0x40000002,0xc0000040,0xc0000002,0x80000000,0x80000000,0x80000002,0x00000040,0x00000002,0x80000000,0x80000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000105,0x00000089,0x00000016,0x0000020b,0x0000011b,0x0000012d,0x0000041e,0x00000224,0x00000050,0x0000092e,0x0000046c,0x000005b6,0x0000106a,0x00000b90,0x00000152] }, // II(46,2) + { testt: 58, maskb: 19, dm: [0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d,0x8000041a,0x000002e4] }, // II(47,0) + { testt: 58, maskb: 20, dm: [0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d,0x8000041a] }, // II(48,0) + { testt: 58, maskb: 21, dm: [0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d] }, // II(49,0) + { testt: 58, maskb: 22, dm: [0xf0000010,0xf000006a,0x80000040,0x90000070,0xb0000053,0x30000008,0x00000043,0xd0000072,0xb0000010,0xf0000062,0xc0000042,0x00000030,0xe0000042,0x20000060,0xe0000041,0x20000050,0xc0000041,0xe0000072,0xa0000003,0xc0000012,0x60000041,0xc0000032,0x20000001,0xc0000002,0xe0000042,0x60000042,0x80000002,0x00000000,0x00000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000001,0x00000060,0x80000003,0x40000002,0xc0000040,0xc0000002,0x80000000,0x80000000,0x80000002,0x00000040,0x00000002,0x80000000,0x80000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000105,0x00000089,0x00000016,0x0000020b,0x0000011b,0x0000012d,0x0000041e,0x00000224,0x00000050,0x0000092e,0x0000046c,0x000005b6] }, // II(49,2) + { testt: 65, maskb: 23, dm: [0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b] }, // II(50,0) + { testt: 65, maskb: 24, dm: [0xd0000072,0xf0000010,0xf000006a,0x80000040,0x90000070,0xb0000053,0x30000008,0x00000043,0xd0000072,0xb0000010,0xf0000062,0xc0000042,0x00000030,0xe0000042,0x20000060,0xe0000041,0x20000050,0xc0000041,0xe0000072,0xa0000003,0xc0000012,0x60000041,0xc0000032,0x20000001,0xc0000002,0xe0000042,0x60000042,0x80000002,0x00000000,0x00000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000001,0x00000060,0x80000003,0x40000002,0xc0000040,0xc0000002,0x80000000,0x80000000,0x80000002,0x00000040,0x00000002,0x80000000,0x80000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000105,0x00000089,0x00000016,0x0000020b,0x0000011b,0x0000012d,0x0000041e,0x00000224,0x00000050,0x0000092e,0x0000046c] }, // II(50,2) + { testt: 65, maskb: 25, dm: [0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b] }, // II(51,0) + { testt: 65, maskb: 26, dm: [0x00000043,0xd0000072,0xf0000010,0xf000006a,0x80000040,0x90000070,0xb0000053,0x30000008,0x00000043,0xd0000072,0xb0000010,0xf0000062,0xc0000042,0x00000030,0xe0000042,0x20000060,0xe0000041,0x20000050,0xc0000041,0xe0000072,0xa0000003,0xc0000012,0x60000041,0xc0000032,0x20000001,0xc0000002,0xe0000042,0x60000042,0x80000002,0x00000000,0x00000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000001,0x00000060,0x80000003,0x40000002,0xc0000040,0xc0000002,0x80000000,0x80000000,0x80000002,0x00000040,0x00000002,0x80000000,0x80000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000105,0x00000089,0x00000016,0x0000020b,0x0000011b,0x0000012d,0x0000041e,0x00000224,0x00000050,0x0000092e] }, // II(51,2) + { testt: 65, maskb: 27, dm: [0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014] }, // II(52,0) + { testt: 65, maskb: 28, dm: [0xcc000014,0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089] }, // II(53,0) + { testt: 65, maskb: 29, dm: [0x0400001c,0xcc000014,0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107] }, // II(54,0) + { testt: 65, maskb: 30, dm: [0x00000010,0x0400001c,0xcc000014,0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b] }, // II(55,0) + { testt: 65, maskb: 31, dm: [0x2600001a,0x00000010,0x0400001c,0xcc000014,0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046] }, // II(56,0) +]; + +/** + * Check the unavoidable bit conditions for every DV against an expanded + * message block. Returns a mask whose bit `maskb` is set when every UBC for + * that DV holds, i.e. when the DV is worth the cost of a recompression check. + */ +export function ubcCheck(W: Int32Array): number { + let mask = ~0; + + + mask &= (((((W[44]^W[45])>>>29)&1)-1) | ~((1 << 7)|(1 << 13)|(1 << 15)|(1 << 16)|(1 << 17)|(1 << 23)|(1 << 25))); + mask &= (((((W[49]^W[50])>>>29)&1)-1) | ~((1 << 3)|(1 << 16)|(1 << 23)|(1 << 25)|(1 << 30)|(1 << 31))); + mask &= (((((W[48]^W[49])>>>29)&1)-1) | ~((1 << 2)|(1 << 15)|(1 << 21)|(1 << 23)|(1 << 29)|(1 << 30))); + mask &= ((((W[47]^(W[50]>>>25))&(1<<4))-(1<<4)) | ~((1 << 5)|(1 << 9)|(1 << 13)|(1 << 16)|(1 << 25)|(1 << 31))); + mask &= (((((W[47]^W[48])>>>29)&1)-1) | ~((1 << 1)|(1 << 13)|(1 << 20)|(1 << 21)|(1 << 28)|(1 << 29))); + mask &= (((((W[46]>>>4)^(W[49]>>>29))&1)-1) | ~((1 << 3)|(1 << 7)|(1 << 11)|(1 << 15)|(1 << 23)|(1 << 30))); + mask &= (((((W[46]^W[47])>>>29)&1)-1) | ~((1 << 0)|(1 << 11)|(1 << 19)|(1 << 20)|(1 << 27)|(1 << 28))); + mask &= (((((W[45]>>>4)^(W[48]>>>29))&1)-1) | ~((1 << 2)|(1 << 5)|(1 << 9)|(1 << 13)|(1 << 21)|(1 << 29))); + mask &= (((((W[45]^W[46])>>>29)&1)-1) | ~((1 << 9)|(1 << 15)|(1 << 17)|(1 << 19)|(1 << 25)|(1 << 27))); + mask &= (((((W[44]>>>4)^(W[47]>>>29))&1)-1) | ~((1 << 1)|(1 << 3)|(1 << 7)|(1 << 11)|(1 << 20)|(1 << 28))); + mask &= (((((W[43]>>>4)^(W[46]>>>29))&1)-1) | ~((1 << 0)|(1 << 2)|(1 << 5)|(1 << 9)|(1 << 19)|(1 << 27))); + mask &= (((((W[43]^W[44])>>>29)&1)-1) | ~((1 << 5)|(1 << 11)|(1 << 13)|(1 << 16)|(1 << 21)|(1 << 23))); + mask &= (((((W[42]>>>4)^(W[45]>>>29))&1)-1) | ~((1 << 1)|(1 << 3)|(1 << 7)|(1 << 15)|(1 << 17)|(1 << 25))); + mask &= (((((W[41]>>>4)^(W[44]>>>29))&1)-1) | ~((1 << 0)|(1 << 2)|(1 << 5)|(1 << 13)|(1 << 16)|(1 << 23))); + mask &= (((((W[40]^W[41])>>>29)&1)-1) | ~((1 << 1)|(1 << 5)|(1 << 7)|(1 << 17)|(1 << 19)|(1 << 31))); + mask &= (((((W[54]^W[55])>>>29)&1)-1) | ~((1 << 13)|(1 << 19)|(1 << 23)|(1 << 30)|(1 << 31))); + mask &= (((((W[53]^W[54])>>>29)&1)-1) | ~((1 << 11)|(1 << 17)|(1 << 21)|(1 << 29)|(1 << 30))); + mask &= (((((W[52]^W[53])>>>29)&1)-1) | ~((1 << 9)|(1 << 16)|(1 << 20)|(1 << 28)|(1 << 29))); + mask &= ((((W[50]^(W[53]>>>25))&(1<<4))-(1<<4)) | ~((1 << 11)|(1 << 15)|(1 << 17)|(1 << 20)|(1 << 29))); + mask &= (((((W[50]^W[51])>>>29)&1)-1) | ~((1 << 5)|(1 << 17)|(1 << 25)|(1 << 27)|(1 << 31))); + mask &= ((((W[49]^(W[52]>>>25))&(1<<4))-(1<<4)) | ~((1 << 9)|(1 << 13)|(1 << 16)|(1 << 19)|(1 << 28))); + mask &= ((((W[48]^(W[51]>>>25))&(1<<4))-(1<<4)) | ~((1 << 7)|(1 << 11)|(1 << 15)|(1 << 17)|(1 << 27))); + mask &= (((((W[42]^W[43])>>>29)&1)-1) | ~((1 << 3)|(1 << 9)|(1 << 11)|(1 << 20)|(1 << 21))); + mask &= (((((W[41]^W[42])>>>29)&1)-1) | ~((1 << 2)|(1 << 7)|(1 << 9)|(1 << 19)|(1 << 20))); + mask &= (((((W[40]>>>4)^(W[43]>>>29))&1)-1) | ~((1 << 1)|(1 << 3)|(1 << 11)|(1 << 21)|(1 << 31))); + mask &= (((((W[39]>>>4)^(W[42]>>>29))&1)-1) | ~((1 << 0)|(1 << 2)|(1 << 9)|(1 << 20)|(1 << 30))); + if (mask & ((1 << 1)|(1 << 7)|(1 << 19)|(1 << 29)|(1 << 31))) + mask &= (((((W[38]>>>4)^(W[41]>>>29))&1)-1) | ~((1 << 1)|(1 << 7)|(1 << 19)|(1 << 29)|(1 << 31))); + mask &= (((((W[37]>>>4)^(W[40]>>>29))&1)-1) | ~((1 << 0)|(1 << 5)|(1 << 17)|(1 << 28)|(1 << 30))); + if (mask & ((1 << 15)|(1 << 20)|(1 << 25)|(1 << 31))) + mask &= (((((W[55]^W[56])>>>29)&1)-1) | ~((1 << 15)|(1 << 20)|(1 << 25)|(1 << 31))); + if (mask & ((1 << 15)|(1 << 20)|(1 << 23)|(1 << 31))) + mask &= ((((W[52]^(W[55]>>>25))&(1<<4))-(1<<4)) | ~((1 << 15)|(1 << 20)|(1 << 23)|(1 << 31))); + if (mask & ((1 << 13)|(1 << 19)|(1 << 21)|(1 << 30))) + mask &= ((((W[51]^(W[54]>>>25))&(1<<4))-(1<<4)) | ~((1 << 13)|(1 << 19)|(1 << 21)|(1 << 30))); + if (mask & ((1 << 7)|(1 << 19)|(1 << 27)|(1 << 28))) + mask &= (((((W[51]^W[52])>>>29)&1)-1) | ~((1 << 7)|(1 << 19)|(1 << 27)|(1 << 28))); + if (mask & ((1 << 3)|(1 << 9)|(1 << 16)|(1 << 20))) + mask &= (((((W[36]>>>4)^(W[40]>>>29))&1)-1) | ~((1 << 3)|(1 << 9)|(1 << 16)|(1 << 20))); + if (mask & ((1 << 15)|(1 << 20)|(1 << 21))) + mask &= ((0-(((W[53]^W[56])>>>29)&1)) | ~((1 << 15)|(1 << 20)|(1 << 21))); + if (mask & ((1 << 11)|(1 << 17)|(1 << 19))) + mask &= ((0-(((W[51]^W[54])>>>29)&1)) | ~((1 << 11)|(1 << 17)|(1 << 19))); + if (mask & ((1 << 9)|(1 << 13)|(1 << 16))) + mask &= ((0-(((W[50]^W[52])>>>29)&1)) | ~((1 << 9)|(1 << 13)|(1 << 16))); + if (mask & ((1 << 7)|(1 << 11)|(1 << 15))) + mask &= ((0-(((W[49]^W[51])>>>29)&1)) | ~((1 << 7)|(1 << 11)|(1 << 15))); + if (mask & ((1 << 5)|(1 << 9)|(1 << 13))) + mask &= ((0-(((W[48]^W[50])>>>29)&1)) | ~((1 << 5)|(1 << 9)|(1 << 13))); + if (mask & ((1 << 3)|(1 << 7)|(1 << 11))) + mask &= ((0-(((W[47]^W[49])>>>29)&1)) | ~((1 << 3)|(1 << 7)|(1 << 11))); + if (mask & ((1 << 2)|(1 << 5)|(1 << 9))) + mask &= ((0-(((W[46]^W[48])>>>29)&1)) | ~((1 << 2)|(1 << 5)|(1 << 9))); + mask &= ((((W[45]^W[47])&(1<<6))-(1<<6)) | ~((1 << 6)|(1 << 10)|(1 << 14))); + if (mask & ((1 << 1)|(1 << 3)|(1 << 7))) + mask &= ((0-(((W[45]^W[47])>>>29)&1)) | ~((1 << 1)|(1 << 3)|(1 << 7))); + mask &= (((((W[44]^W[46])>>>6)&1)-1) | ~((1 << 4)|(1 << 8)|(1 << 12))); + if (mask & ((1 << 0)|(1 << 2)|(1 << 5))) + mask &= ((0-(((W[44]^W[46])>>>29)&1)) | ~((1 << 0)|(1 << 2)|(1 << 5))); + mask &= ((0-((W[41]^(W[42]>>>5))&(1<<1))) | ~((1 << 8)|(1 << 18)|(1 << 26))); + mask &= ((0-((W[40]^(W[41]>>>5))&(1<<1))) | ~((1 << 6)|(1 << 14)|(1 << 24))); + if (mask & ((1 << 1)|(1 << 3)|(1 << 31))) + mask &= ((0-(((W[40]^W[42])>>>4)&1)) | ~((1 << 1)|(1 << 3)|(1 << 31))); + mask &= ((0-((W[39]^(W[40]>>>5))&(1<<1))) | ~((1 << 4)|(1 << 12)|(1 << 22))); + if (mask & ((1 << 0)|(1 << 2)|(1 << 30))) + mask &= ((0-(((W[39]^W[41])>>>4)&1)) | ~((1 << 0)|(1 << 2)|(1 << 30))); + if (mask & ((1 << 1)|(1 << 29)|(1 << 31))) + mask &= ((0-(((W[38]^W[40])>>>4)&1)) | ~((1 << 1)|(1 << 29)|(1 << 31))); + if (mask & ((1 << 0)|(1 << 28)|(1 << 30))) + mask &= ((0-(((W[37]^W[39])>>>4)&1)) | ~((1 << 0)|(1 << 28)|(1 << 30))); + mask &= ((0-((W[36]^(W[37]>>>5))&(1<<1))) | ~((1 << 6)|(1 << 12)|(1 << 18))); + if (mask & ((1 << 2)|(1 << 7)|(1 << 19))) + mask &= (((((W[35]>>>4)^(W[39]>>>29))&1)-1) | ~((1 << 2)|(1 << 7)|(1 << 19))); + if (mask & ((1 << 7)|(1 << 20))) + mask &= ((0-((W[63]^(W[64]>>>5))&(1<<0))) | ~((1 << 7)|(1 << 20))); + if (mask & ((1 << 2)|(1 << 16))) + mask &= ((0-((W[63]^(W[64]>>>5))&(1<<1))) | ~((1 << 2)|(1 << 16))); + if (mask & ((1 << 5)|(1 << 19))) + mask &= ((0-((W[62]^(W[63]>>>5))&(1<<0))) | ~((1 << 5)|(1 << 19))); + if (mask & ((1 << 3)|(1 << 17))) + mask &= ((0-((W[61]^(W[62]>>>5))&(1<<0))) | ~((1 << 3)|(1 << 17))); + mask &= ((0-((W[61]^(W[62]>>>5))&(1<<2))) | ~((1 << 4)|(1 << 18))); + if (mask & ((1 << 2)|(1 << 16))) + mask &= ((0-((W[60]^(W[61]>>>5))&(1<<0))) | ~((1 << 2)|(1 << 16))); + if (mask & ((1 << 25)|(1 << 29))) + mask &= (((((W[58]^W[59])>>>29)&1)-1) | ~((1 << 25)|(1 << 29))); + if (mask & ((1 << 23)|(1 << 28))) + mask &= (((((W[57]^W[58])>>>29)&1)-1) | ~((1 << 23)|(1 << 28))); + if (mask & ((1 << 27)|(1 << 29))) + mask &= ((((W[56]^(W[59]>>>25))&(1<<4))-(1<<4)) | ~((1 << 27)|(1 << 29))); + if (mask & ((1 << 25)|(1 << 27))) + mask &= ((0-(((W[56]^W[59])>>>29)&1)) | ~((1 << 25)|(1 << 27))); + if (mask & ((1 << 21)|(1 << 27))) + mask &= (((((W[56]^W[57])>>>29)&1)-1) | ~((1 << 21)|(1 << 27))); + if (mask & ((1 << 25)|(1 << 28))) + mask &= ((((W[55]^(W[58]>>>25))&(1<<4))-(1<<4)) | ~((1 << 25)|(1 << 28))); + if (mask & ((1 << 23)|(1 << 27))) + mask &= ((((W[54]^(W[57]>>>25))&(1<<4))-(1<<4)) | ~((1 << 23)|(1 << 27))); + if (mask & ((1 << 21)|(1 << 25))) + mask &= ((((W[53]^(W[56]>>>25))&(1<<4))-(1<<4)) | ~((1 << 21)|(1 << 25))); + mask &= ((((W[51]^(W[50]>>>5))&(1<<1))-(1<<1)) | ~((1 << 12)|(1 << 18))); + mask &= ((((W[48]^W[50])&(1<<6))-(1<<6)) | ~((1 << 12)|(1 << 18))); + if (mask & ((1 << 13)|(1 << 15))) + mask &= ((0-(((W[48]^W[55])>>>29)&1)) | ~((1 << 13)|(1 << 15))); + mask &= ((((W[47]^W[49])&(1<<6))-(1<<6)) | ~((1 << 10)|(1 << 14))); + mask &= ((((W[48]^(W[47]>>>5))&(1<<1))-(1<<1)) | ~((1 << 6)|(1 << 26))); + mask &= ((((W[46]^W[48])&(1<<6))-(1<<6)) | ~((1 << 8)|(1 << 12))); + mask &= ((((W[47]^(W[46]>>>5))&(1<<1))-(1<<1)) | ~((1 << 4)|(1 << 24))); + mask &= ((0-((W[44]^(W[45]>>>5))&(1<<1))) | ~((1 << 14)|(1 << 22))); + mask &= ((((W[43]^W[45])&(1<<6))-(1<<6)) | ~((1 << 6)|(1 << 10))); + mask &= (((((W[42]^W[44])>>>6)&1)-1) | ~((1 << 4)|(1 << 8))); + mask &= ((((W[43]^(W[42]>>>5))&(1<<1))-(1<<1)) | ~((1 << 18)|(1 << 26))); + mask &= ((((W[42]^(W[41]>>>5))&(1<<1))-(1<<1)) | ~((1 << 14)|(1 << 24))); + mask &= ((((W[41]^(W[40]>>>5))&(1<<1))-(1<<1)) | ~((1 << 12)|(1 << 22))); + if (mask & ((1 << 15)|(1 << 25))) + mask &= ((((W[39]^(W[43]>>>25))&(1<<4))-(1<<4)) | ~((1 << 15)|(1 << 25))); + if (mask & ((1 << 13)|(1 << 23))) + mask &= ((((W[38]^(W[42]>>>25))&(1<<4))-(1<<4)) | ~((1 << 13)|(1 << 23))); + if (mask & ((1 << 8)|(1 << 14))) + mask &= ((0-((W[37]^(W[38]>>>5))&(1<<1))) | ~((1 << 8)|(1 << 14))); + if (mask & ((1 << 11)|(1 << 21))) + mask &= ((((W[37]^(W[41]>>>25))&(1<<4))-(1<<4)) | ~((1 << 11)|(1 << 21))); + if (mask & ((1 << 27)|(1 << 29))) + mask &= ((0-((W[36]^W[38])&(1<<4))) | ~((1 << 27)|(1 << 29))); + mask &= ((0-((W[35]^(W[36]>>>5))&(1<<1))) | ~((1 << 4)|(1 << 10))); + if (mask & ((1 << 13)|(1 << 19))) + mask &= ((((W[35]^(W[39]>>>25))&(1<<3))-(1<<3)) | ~((1 << 13)|(1 << 19))); +if (mask) { + + if (mask & (1 << 0)) + if ( + !((W[61]^(W[62]>>>5)) & (1<<1)) + || !(!((W[59]^(W[63]>>>25)) & (1<<5))) + || !((W[58]^(W[63]>>>30)) & (1<<0)) + ) mask &= ~(1 << 0); + if (mask & (1 << 1)) + if ( + !((W[62]^(W[63]>>>5)) & (1<<1)) + || !(!((W[60]^(W[64]>>>25)) & (1<<5))) + || !((W[59]^(W[64]>>>30)) & (1<<0)) + ) mask &= ~(1 << 1); + if (mask & (1 << 4)) + mask &= ((~((W[40]^W[42])>>>2)) | ~(1 << 4)); + if (mask & (1 << 6)) + if ( + !((W[62]^(W[63]>>>5)) & (1<<2)) + || !(!((W[41]^W[43]) & (1<<6))) + ) mask &= ~(1 << 6); + if (mask & (1 << 8)) + if ( + !((W[63]^(W[64]>>>5)) & (1<<2)) + || !(!((W[48]^(W[49]<<5)) & (1<<6))) + ) mask &= ~(1 << 8); + if (mask & (1 << 10)) + if ( + !(!((W[49]^(W[50]<<5)) & (1<<6))) + || !((W[42]^W[50]) & (1<<1)) + || !(!((W[39]^(W[40]<<5)) & (1<<6))) + || !((W[38]^W[40]) & (1<<1)) + ) mask &= ~(1 << 10); + if (mask & (1 << 11)) + mask &= ((((W[36]^W[37])<<7)) | ~(1 << 11)); + if (mask & (1 << 12)) + mask &= ((((W[43]^W[51])<<11)) | ~(1 << 12)); + if (mask & (1 << 13)) + mask &= ((((W[37]^W[38])<<9)) | ~(1 << 13)); + if (mask & (1 << 14)) + if ( + !(!((W[51]^(W[52]<<5)) & (1<<6))) + || !(!((W[49]^W[51]) & (1<<6))) + || !(!((W[37]^(W[37]>>>5)) & (1<<1))) + || !(!((W[35]^(W[39]>>>25)) & (1<<5))) + ) mask &= ~(1 << 14); + if (mask & (1 << 15)) + mask &= ((((W[38]^W[39])<<11)) | ~(1 << 15)); + if (mask & (1 << 18)) + mask &= ((((W[47]^W[51])<<17)) | ~(1 << 18)); + if (mask & (1 << 20)) + if ( + !(!((W[36]^(W[40]>>>25)) & (1<<3))) + || !((W[35]^(W[40]<<2)) & (1<<30)) + ) mask &= ~(1 << 20); + if (mask & (1 << 21)) + if ( + !(!((W[37]^(W[41]>>>25)) & (1<<3))) + || !((W[36]^(W[41]<<2)) & (1<<30)) + ) mask &= ~(1 << 21); + if (mask & (1 << 22)) + if ( + !(!((W[53]^(W[54]<<5)) & (1<<6))) + || !(!((W[51]^W[53]) & (1<<6))) + || !((W[50]^W[54]) & (1<<1)) + || !(!((W[45]^(W[46]<<5)) & (1<<6))) + || !(!((W[37]^(W[41]>>>25)) & (1<<5))) + || !((W[36]^(W[41]>>>30)) & (1<<0)) + ) mask &= ~(1 << 22); + if (mask & (1 << 23)) + if ( + !((W[55]^W[58]) & (1<<29)) + || !(!((W[38]^(W[42]>>>25)) & (1<<3))) + || !((W[37]^(W[42]<<2)) & (1<<30)) + ) mask &= ~(1 << 23); + if (mask & (1 << 24)) + if ( + !(!((W[54]^(W[55]<<5)) & (1<<6))) + || !(!((W[52]^W[54]) & (1<<6))) + || !((W[51]^W[55]) & (1<<1)) + || !((W[45]^W[47]) & (1<<1)) + || !(!((W[38]^(W[42]>>>25)) & (1<<5))) + || !((W[37]^(W[42]>>>30)) & (1<<0)) + ) mask &= ~(1 << 24); + if (mask & (1 << 25)) + if ( + !(!((W[39]^(W[43]>>>25)) & (1<<3))) + || !((W[38]^(W[43]<<2)) & (1<<30)) + ) mask &= ~(1 << 25); + if (mask & (1 << 26)) + if ( + !(!((W[55]^(W[56]<<5)) & (1<<6))) + || !(!((W[53]^W[55]) & (1<<6))) + || !((W[52]^W[56]) & (1<<1)) + || !((W[46]^W[48]) & (1<<1)) + || !(!((W[39]^(W[43]>>>25)) & (1<<5))) + || !((W[38]^(W[43]>>>30)) & (1<<0)) + ) mask &= ~(1 << 26); + if (mask & (1 << 27)) + if ( + !(!((W[59]^W[60]) & (1<<29))) + || !(!((W[40]^(W[44]>>>25)) & (1<<3))) + || !(!((W[40]^(W[44]>>>25)) & (1<<4))) + || !((W[39]^(W[44]<<2)) & (1<<30)) + ) mask &= ~(1 << 27); + if (mask & (1 << 28)) + if ( + !((W[58]^W[61]) & (1<<29)) + || !(!((W[57]^(W[61]>>>25)) & (1<<4))) + || !(!((W[41]^(W[45]>>>25)) & (1<<3))) + || !(!((W[41]^(W[45]>>>25)) & (1<<4))) + ) mask &= ~(1 << 28); + if (mask & (1 << 29)) + if ( + !(!((W[58]^(W[62]>>>25)) & (1<<4))) + || !(!((W[42]^(W[46]>>>25)) & (1<<3))) + || !(!((W[42]^(W[46]>>>25)) & (1<<4))) + ) mask &= ~(1 << 29); + if (mask & (1 << 30)) + if ( + !(!((W[59]^(W[63]>>>25)) & (1<<4))) + || !(!((W[57]^(W[59]>>>25)) & (1<<4))) + || !(!((W[43]^(W[47]>>>25)) & (1<<3))) + || !(!((W[43]^(W[47]>>>25)) & (1<<4))) + ) mask &= ~(1 << 30); + if (mask & (1 << 31)) + if ( + !(!((W[60]^(W[64]>>>25)) & (1<<4))) + || !(!((W[44]^(W[48]>>>25)) & (1<<3))) + || !(!((W[44]^(W[48]>>>25)) & (1<<4))) + ) mask &= ~(1 << 31); +} + return mask; +} diff --git a/src/git/snapshot.ts b/src/git/snapshot.ts @@ -10,6 +10,22 @@ } +function safePath(p: string): boolean { + if (!p || p.startsWith("/") || p.includes("\\")) return false; + return p.split("/").every((c) => c !== "" && c !== "." && c !== ".."); +} + +function safeTarget(t: string): boolean { + if (t.startsWith("/")) return false; + return t.split("/").every((c) => c !== ".."); +} + +function safeEntry(f: SnapshotFile): boolean { + if (!safePath(f.path)) return false; + if (f.symlink && !safeTarget(new TextDecoder().decode(f.data))) return false; + return true; +} + function octal(n: number, width: number): Uint8Array { const s = n.toString(8).padStart(width - 1, "0") + "\0"; return te.encode(s); @@ -49,6 +65,7 @@ export function tarGz(prefix: string, files: SnapshotFile[], mtime: number): Uint8Array { const parts: Uint8Array[] = []; for (const f of files) { + if (!safeEntry(f)) continue; const path = `${prefix}/${f.path}`; if (f.symlink) { const target = new TextDecoder().decode(f.data); @@ -94,6 +111,7 @@ const central: Uint8Array[] = []; let offset = 0; for (const f of files) { + if (!safeEntry(f)) continue; const name = te.encode(`${prefix}/${f.path}`); const crc = crc32(f.data); const compressed = f.data.length ? pako.deflateRaw(f.data) : new Uint8Array(0); @@ -136,8 +154,8 @@ const eocd = new Uint8Array(22); const ev = new DataView(eocd.buffer); ev.setUint32(0, 0x06054b50, true); - ev.setUint16(8, files.length, true); - ev.setUint16(10, files.length, true); + ev.setUint16(8, central.length, true); + ev.setUint16(10, central.length, true); ev.setUint32(12, cdLen, true); ev.setUint32(16, cdStart, true); return concat([...parts, ...central, eocd]); diff --git a/src/git/store.ts b/src/git/store.ts @@ -1,6 +1,7 @@ -import { concat } from "./util"; +import { concat, te, sha1hex } from "./util"; import { deflate, inflate } from "./zlib"; import { ObjType } from "./objects"; +import { OidSet } from "./oidset"; import { PackStore, ObjCache, ObjRec } from "./packstore"; const CHUNK = 1024 * 1024; // stay well under the DO SQLite per-row limit @@ -16,6 +17,15 @@ export class GitStore { readonly packs: PackStore; readonly cache = new ObjCache(CACHE_BUDGET); + /** + * Whether the loose `objects` table holds anything. A pushed repo is + * pack-native — loose objects only appear after a gc migration (see + * runGc) — so on the normal path this is false and every get/has/typeAndSize + * skips a guaranteed-miss loose SELECT and goes straight to the pack index. + * Cached in memory and mirrored to the `has-loose` meta row so it survives + * isolate eviction; back-filled once for repos that predate the flag. + */ + private hasLoose: boolean; constructor(private sql: SqlStorage) { this.packs = new PackStore(sql, (oid) => this.getLoose(oid)); @@ -39,22 +49,49 @@ key TEXT PRIMARY KEY, value TEXT NOT NULL ); + CREATE TABLE IF NOT EXISTS reachable ( + oid TEXT PRIMARY KEY + ); + CREATE TABLE IF NOT EXISTS peeled ( + oid TEXT PRIMARY KEY, + target TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS stats_cache ( + tip_oid TEXT NOT NULL, + period TEXT NOT NULL, + data TEXT NOT NULL, + PRIMARY KEY (tip_oid, period) + ); `); + const flag = this.getMeta("has-loose"); + if (flag === null) { + // back-fill: a repo migrated by an older build may already hold loose + // objects with no flag set — probe once and record the answer + const any = this.sql.exec("SELECT 1 FROM objects LIMIT 1").toArray().length > 0; + this.setMeta("has-loose", any ? "1" : "0"); + this.hasLoose = any; + } else { + this.hasLoose = flag === "1"; + } } has(oid: string): boolean { - if (this.sql.exec("SELECT 1 FROM objects WHERE oid = ?", oid).toArray().length > 0) return true; + if (this.hasLoose && this.sql.exec("SELECT 1 FROM objects WHERE oid = ?", oid).toArray().length > 0) return true; return this.packs.typeAndSize(oid) !== null; } typeAndSize(oid: string): { type: ObjType; size: number } | null { - const rows = this.sql - .exec<{ type: ObjType; size: number }>("SELECT type, size FROM objects WHERE oid = ?", oid) - .toArray(); - return rows[0] ?? this.packs.typeAndSize(oid); + if (this.hasLoose) { + const rows = this.sql + .exec<{ type: ObjType; size: number }>("SELECT type, size FROM objects WHERE oid = ?", oid) + .toArray(); + if (rows[0]) return rows[0]; + } + return this.packs.typeAndSize(oid); } private getLoose(oid: string): ObjRec | null { + if (!this.hasLoose) return null; const rows = this.sql .exec<{ type: ObjType; size: number }>("SELECT type, size FROM objects WHERE oid = ?", oid) .toArray(); @@ -78,6 +115,12 @@ const slice = packed.slice(off, off + CHUNK); this.sql.exec("INSERT INTO chunks (oid, seq, data) VALUES (?, ?, ?)", oid, seq, slice.buffer); } + // the repo now has at least one loose object (gc migration path): the + // loose probe can no longer be skipped + if (!this.hasLoose) { + this.hasLoose = true; + this.setMeta("has-loose", "1"); + } } objectCount(): number { @@ -87,10 +130,12 @@ /** Resolve an abbreviated oid; null if unknown or ambiguous. */ findOid(prefix: string): string | null { if (!/^[0-9a-f]{4,40}$/.test(prefix)) return null; - const loose = this.sql - .exec<{ oid: string }>("SELECT oid FROM objects WHERE oid LIKE ? LIMIT 2", prefix + "%") - .toArray() - .map((r) => r.oid); + const loose = this.hasLoose + ? this.sql + .exec<{ oid: string }>("SELECT oid FROM objects WHERE oid LIKE ? LIMIT 2", prefix + "%") + .toArray() + .map((r) => r.oid) + : []; const all = [...new Set([...loose, ...this.packs.findOidPrefix(prefix)])]; return all.length === 1 ? all[0] : null; } @@ -109,10 +154,12 @@ } wipe(): void { - for (const t of ["objects", "chunks", "refs", "meta"]) { + for (const t of ["objects", "chunks", "refs", "meta", "reachable", "peeled", "stats_cache"]) { this.sql.exec(`DROP TABLE IF EXISTS ${t}`); } this.packs.wipe(); + // meta (with the flag) is gone; a fresh empty repo has no loose objects + this.hasLoose = false; } @@ -175,4 +222,109 @@ value ); } + + /** + * Cached peel target for an oid (tag -> ... -> commit). Tag oids are + * content-addressed and immutable, so a hit never needs a version check. + */ + getPeeled(oid: string): string | null { + const rows = this.sql.exec<{ target: string }>("SELECT target FROM peeled WHERE oid = ?", oid).toArray(); + return rows[0]?.target ?? null; + } + + setPeeled(oid: string, target: string): void { + this.sql.exec("INSERT OR IGNORE INTO peeled (oid, target) VALUES (?, ?)", oid, target); + } + + /** + * Cached commit-activity aggregation for a (tip commit oid, period) pair. + * Keyed on the tip's oid rather than a ref name: history under a fixed + * commit oid is immutable, so no version check is needed for a hit. + */ + getStatsCache(tipOid: string, period: string): string | null { + const rows = this.sql + .exec<{ data: string }>("SELECT data FROM stats_cache WHERE tip_oid = ? AND period = ?", tipOid, period) + .toArray(); + return rows[0]?.data ?? null; + } + + setStatsCache(tipOid: string, period: string, data: string): void { + this.sql.exec( + "INSERT OR REPLACE INTO stats_cache (tip_oid, period, data) VALUES (?, ?, ?)", + tipOid, + period, + data + ); + } + + /** + * A tag over the current ref state (names + tips + HEAD). The cached + * reachable object set is keyed on it: any push, ref delete, or gc changes + * a tip and so bumps the version, which is exactly when the set can change. + * A sha1 hex string never collides with the "" invalidation sentinel. + */ + reachableVersion(): string { + const parts: string[] = []; + for (const r of this.refs()) parts.push(`${r.name}\0${r.target}`); + parts.push(`HEAD\0${this.getRef("HEAD") ?? ""}`); + return sha1hex(te.encode(parts.join("\n"))); + } + + /** + * The cached reachable object set as an OidSet with every entry marked (so + * markedSize/markedAtHex/isMarkedHex feed the pack-emission path directly), + * or null when the cache is missing or its version no longer matches current + * refs. The version check is the only correctness gate — a stale set is + * never returned, so the fast path can never serve unreachable objects. + */ + loadReachable(): OidSet | null { + if (this.getMeta("reachable-version") !== this.reachableVersion()) return null; + const rows = this.sql.exec<{ oid: string }>("SELECT oid FROM reachable").toArray(); + if (!rows.length) return null; + const set = new OidSet(rows.length); + for (const r of rows) set.markHex(r.oid); + return set; + } + + /** + * Persist the marked sub-set of `set` (a full clone's reachable object set) + * and stamp it with the current ref version. Yields periodically so a large + * write never monopolizes the event loop; if refs change mid-write (a push + * landing during a cold clone) the version guard abandons the stamp, leaving + * the half-written table unversioned and thus ignored until it is rebuilt. + */ + async saveReachable(set: OidSet): Promise<void> { + const version = this.reachableVersion(); + this.sql.exec("DELETE FROM reachable"); + const n = set.markedSize; + for (let i = 0; i < n; i++) { + this.sql.exec("INSERT OR IGNORE INTO reachable (oid) VALUES (?)", set.markedAtHex(i)); + if ((i & 8191) === 8191) { + if (this.reachableVersion() !== version) return; + await new Promise((r) => setTimeout(r, 0)); + } + } + if (this.reachableVersion() !== version) return; + this.setMeta("reachable-version", version); + } + + /** + * Fold every oid in `set` into the cached reachable set and re-stamp it with + * the current ref version. Used on a fast-forward push, where the receive + * connectivity walk already visited exactly the newly connected objects: + * union with the prior (validated) set yields the new reachable set without + * a fresh graph walk. Only sound when no object became unreachable — the + * caller must have a valid prior cache and no stranded history. + */ + extendReachable(set: OidSet): void { + for (let i = 0; i < set.size; i++) { + this.sql.exec("INSERT OR IGNORE INTO reachable (oid) VALUES (?)", set.atHex(i)); + } + this.setMeta("reachable-version", this.reachableVersion()); + } + + /** Drop the version stamp so the next full clone rebuilds the set by walking. */ + invalidateReachable(): void { + this.setMeta("reachable-version", ""); + } } diff --git a/src/git/zlib.ts b/src/git/zlib.ts @@ -1,4 +1,5 @@ import pako from "pako"; +import { concat } from "./util"; /** Deflate with a zlib wrapper (what git uses for loose objects and pack entries). */ export function deflate(data: Uint8Array): Uint8Array { @@ -9,21 +10,24 @@ return pako.inflate(data); } -export function gunzip(data: Uint8Array): Uint8Array { - return pako.ungzip(data); -} - /** - * Inflate one zlib stream that starts at `start` inside `buf`, and report where - * it ended. Pack files concatenate per-object zlib streams with no length - * prefix, so the consumed byte count is the only way to find the next entry. + * gunzip with an output budget. A gzipped request body compresses ~1000:1 in + * the adversarial case, so the compressed length says nothing about what it + * inflates to: the only safe bound is on the bytes coming *out*. Inflating + * incrementally and stopping the moment the budget is passed means an + * over-inflating body costs the budget, not the bomb. */ -export function inflateEntry(buf: Uint8Array, start: number): { data: Uint8Array; end: number } { - const inf = new pako.Inflate(); - inf.push(buf.subarray(start), true); - const anyInf = inf as unknown as { err: number; msg: string; ended: boolean; strm: { avail_in: number } }; - if (anyInf.err) throw new Error(`inflate failed at ${start}: ${anyInf.msg}`); - if (!anyInf.ended) throw new Error(`truncated zlib stream at ${start}`); - const consumed = buf.length - start - anyInf.strm.avail_in; - return { data: inf.result as Uint8Array, end: start + consumed }; +export function gunzipLimited(data: Uint8Array, maxOut: number): Uint8Array { + const inf = new pako.Inflate({ windowBits: 15 + 16 }); + const parts: Uint8Array[] = []; + let out = 0; + (inf as unknown as { onData: (c: Uint8Array) => void }).onData = (c: Uint8Array) => { + out += c.length; + if (out > maxOut) throw new Error(`request body exceeds maximum size (${maxOut} bytes decompressed)`); + parts.push(c); + }; + inf.push(data, true); + const anyInf = inf as unknown as { err: number; msg: string }; + if (anyInf.err) throw new Error(`gunzip failed: ${anyInf.msg}`); + return concat(parts); } diff --git a/src/index.ts b/src/index.ts @@ -22,8 +22,68 @@ }); } -/** HTTP Basic where the password (or username) is one of the configured tokens. */ -function checkAuth(req: Request, env: Env): Response | null { +function tooManyRequests(retryAfterMs: number): Response { + return new Response("too many failed attempts\n", { + status: 429, + headers: { "retry-after": String(Math.ceil(retryAfterMs / 1000)) }, + }); +} + +async function digest(s: string): Promise<Uint8Array> { + return new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(s))); +} + +/** Fixed-length SHA-256 digests, so this leaks nothing about input length. */ +function digestsEqual(a: Uint8Array, b: Uint8Array): boolean { + let diff = a.length ^ b.length; + for (let i = 0; i < Math.max(a.length, b.length); i++) diff |= (a[i] ?? 0) ^ (b[i] ?? 0); + return diff === 0; +} + +function clientIp(req: Request): string { + return ( + req.headers.get("cf-connecting-ip") ?? + req.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? + "unknown" + ); +} + +const RATE_LIMIT_WINDOW_MS = 60_000; +const RATE_LIMIT_MAX = 10; +const RATE_LIMIT_MAX_IPS = 5000; +const authFailures = new Map<string, { count: number; resetAt: number }>(); + +function pruneAuthFailures(now: number): void { + if (authFailures.size <= RATE_LIMIT_MAX_IPS) return; + for (const [ip, e] of authFailures) { + if (e.resetAt <= now) authFailures.delete(ip); + } + while (authFailures.size > RATE_LIMIT_MAX_IPS) { + const oldest = authFailures.keys().next().value; + if (oldest === undefined) break; + authFailures.delete(oldest); + } +} + +function rateLimited(ip: string): number | null { + const e = authFailures.get(ip); + if (!e || e.resetAt <= Date.now()) return null; + return e.count >= RATE_LIMIT_MAX ? e.resetAt - Date.now() : null; +} + +function recordAuthFailure(ip: string): void { + const now = Date.now(); + const e = authFailures.get(ip); + if (!e || e.resetAt <= now) { + authFailures.set(ip, { count: 1, resetAt: now + RATE_LIMIT_WINDOW_MS }); + } else { + e.count++; + } + pruneAuthFailures(now); +} + +/** HTTP Basic where the password (only) is one of the configured tokens. */ +async function checkAuth(req: Request, env: Env): Promise<Response | null> { const valid = tokens(env); if (!valid.length) { return new Response("access is disabled: set the GIT_TOKEN secret\n", { status: 403 }); @@ -30,16 +90,28 @@ } const header = req.headers.get("authorization") ?? ""; if (!header.startsWith("Basic ")) return unauthorized(); - let user = "", pass = ""; + + const ip = clientIp(req); + const retryMs = rateLimited(ip); + if (retryMs !== null) return tooManyRequests(retryMs); + + let pass = ""; try { const decoded = atob(header.slice(6)); const colon = decoded.indexOf(":"); - user = colon === -1 ? decoded : decoded.slice(0, colon); pass = colon === -1 ? "" : decoded.slice(colon + 1); } catch { + recordAuthFailure(ip); return unauthorized(); } - if (!valid.includes(pass) && !valid.includes(user)) return unauthorized(); + + const passDigest = await digest(pass); + let ok = false; + for (const v of valid) ok = digestsEqual(passDigest, await digest(v)) || ok; + if (!ok) { + recordAuthFailure(ip); + return unauthorized(); + } return null; } @@ -55,7 +127,7 @@ async function indexPage(env: Env): Promise<Response> { const registry = env.REGISTRY.getByName("registry"); - const repos = (await registry.list()).filter((r) => !r.priv); + const repos = await registry.list(); let rows = ""; let lastSection: string | null = null; for (const r of repos) { @@ -83,16 +155,29 @@ * Per-isolate memo of registry lookups. Without it every page view — cache * hit or not — funnels through the single Registry DO, which becomes the * global bottleneck under a traffic spike. Staleness window is small and - * only affects metadata (descriptions, versions, the private flag). + * only affects metadata (descriptions, versions, the private flag). Bounded + * and short-TTL'd on misses so enumerating random repo names can't grow it + * unbounded. */ const infoMemo = new Map<string, { at: number; info: RepoInfo | null }>(); const INFO_TTL_MS = 20_000; +const INFO_NEG_TTL_MS = 2_000; +const INFO_MEMO_MAX = 5000; +function pruneInfoMemo(): void { + while (infoMemo.size > INFO_MEMO_MAX) { + const oldest = infoMemo.keys().next().value; + if (oldest === undefined) break; + infoMemo.delete(oldest); + } +} + async function repoInfo(env: Env, repo: string, fresh: boolean): Promise<RepoInfo | null> { const hit = infoMemo.get(repo); - if (!fresh && hit && Date.now() - hit.at < INFO_TTL_MS) return hit.info; + if (!fresh && hit && Date.now() - hit.at < (hit.info ? INFO_TTL_MS : INFO_NEG_TTL_MS)) return hit.info; const info = await env.REGISTRY.getByName("registry").get(repo); infoMemo.set(repo, { at: Date.now(), info }); + pruneInfoMemo(); return info; } @@ -165,7 +250,7 @@ // pushes, admin operations, and everything on a private repo require auth if (isReceive || isAdmin || info?.priv) { - const denied = checkAuth(req, env); + const denied = await checkAuth(req, env); if (denied) return denied; } diff --git a/src/registry.ts b/src/registry.ts @@ -34,6 +34,7 @@ idle INTEGER NOT NULL DEFAULT 0, ver INTEGER NOT NULL DEFAULT 1 ); + CREATE INDEX IF NOT EXISTS idx_repos_section_name ON repos (section, name); `); // upgrade path for databases created by earlier versions for (const col of ["section TEXT NOT NULL DEFAULT ''", "priv INTEGER NOT NULL DEFAULT 0", "ver INTEGER NOT NULL DEFAULT 1"]) { @@ -80,9 +81,13 @@ return rows[0] ?? null; } - list(): RepoInfo[] { + /** Public index page feed: excludes private repos, bounded so a large table can't dump wholesale over RPC. */ + list(limit = 1000): RepoInfo[] { return this.ctx.storage.sql - .exec<RepoInfo>("SELECT name, desc, owner, section, priv, idle, ver FROM repos ORDER BY section, name") + .exec<RepoInfo>( + "SELECT name, desc, owner, section, priv, idle, ver FROM repos WHERE priv = 0 ORDER BY section, name LIMIT ?", + limit + ) .toArray(); } } diff --git a/src/repo.ts b/src/repo.ts @@ -3,7 +3,8 @@ import { td, te, isOid, concat } from "./git/util"; import { GitStore } from "./git/store"; import { ObjCache } from "./git/packstore"; -import { gunzip } from "./git/zlib"; +import { OidSet } from "./git/oidset"; +import { gunzipLimited } from "./git/zlib"; import { advertisement, uploadPack, @@ -34,8 +35,20 @@ const MAX_DIFF_FILES = 100; const MAX_DIFF_BLOB = 512 * 1024; const MAX_LOG_SCAN = 5000; +/** a path-filtered log rejects most commits, so bound its walk tighter */ +const PATH_LOG_SCAN = 2000; const MAX_STATS_SCAN = 2000; -const MAX_SNAPSHOT_BYTES = 64 * 1024 * 1024; +/** whole-file blame is memory-bound: cap the tip blob and the history bytes */ +const MAX_BLAME_BYTES = 1024 * 1024; +const MAX_BLAME_HISTORY_BYTES = 16 * 1024 * 1024; +// full streaming is a separate follow-up; until then this must fit the isolate +const MAX_SNAPSHOT_BYTES = 16 * 1024 * 1024; +/** the DO SQLite storage ceiling gc must not cross while duplicating objects */ +const DO_STORAGE_CAP = 10 * 1024 * 1024 * 1024; +/** default push cap when MAX_PUSH_MB is unset, in MiB (matches env.ts docs) */ +const DEFAULT_MAX_PUSH_MB = 512; +/** an upload-pack request is wants/haves/caps only — a few hundred KB at most */ +const MAX_UPLOAD_PACK_BYTES = 16 * 1024 * 1024; // like cgit's about-file: a dedicated about page wins over the README const README_NAMES = [ "about.md", @@ -103,6 +116,7 @@ }); } if (path === "/git-upload-pack" && req.method === "POST") { + if (overDeclaredLength(req, MAX_UPLOAD_PACK_BYTES)) return tooLargeResponse(MAX_UPLOAD_PACK_BYTES); if (this.activeUploads >= 4) { return new Response("busy: too many concurrent fetches, retry shortly\n", { status: 503, @@ -109,11 +123,27 @@ headers: { "retry-after": "15" }, }); } + // the slot must span the whole response stream, not just uploadPack's + // return: decrementing when the ReadableStream is handed back — before + // a byte is produced — disables admission control and lets celld + // idle-evict the cell mid-clone. release() fires exactly once, from the + // stream's close/cancel or a non-stream early return. this.activeUploads++; - try { - return await uploadPack(this.store, await this.readBody(req)); - } finally { + let released = false; + const release = () => { + if (released) return; + released = true; this.activeUploads--; + }; + try { + return await uploadPack(this.store, await this.readBody(req), release); + } catch (err) { + release(); + // an over-inflating body is the client's fault, not a server error + if (err instanceof Error && err.message.includes("exceeds maximum size")) { + return tooLargeResponse(MAX_UPLOAD_PACK_BYTES); + } + throw err; } } if (path === "/git-receive-pack" && req.method === "POST") { @@ -127,19 +157,22 @@ return this.handleConfig(JSON.stringify({ description: (await req.text()).trim() })); } if ((path === "/" || path === "") && req.method === "DELETE") { - // wipe() drops every table we own; deliberately NOT storage.deleteAll(): - // on celld, deleteAll sweeps the ltx replication control tables too and - // permanently breaks WAL capture for the cell (patch submitted upstream) - this.store.wipe(); - await this.ctx.storage.deleteAlarm(); - // this instance stays resident: bring the (empty) schema back so - // later requests — including a re-creating push — find their tables - this.store = new GitStore(this.ctx.storage.sql); + // the whole wipe must be atomic: a concurrent request landing between + // the table drop and the re-init hits missing tables and 500s + await this.ctx.blockConcurrencyWhile(async () => { + // wipe() drops every table we own; deliberately NOT storage.deleteAll(): + // on celld, deleteAll sweeps the ltx replication control tables too and + // permanently breaks WAL capture for the cell (patch submitted upstream) + this.store.wipe(); + await this.ctx.storage.deleteAlarm(); + // this instance stays resident: bring the (empty) schema back so + // later requests — including a re-creating push — find their tables + this.store = new GitStore(this.ctx.storage.sql); + }); return new Response("deleted\n"); } if (path === "/gc" && req.method === "POST") { - const result = this.runGc(); - return Response.json(result); + return Response.json(await this.gc()); } if (req.method !== "GET") return new Response("method not allowed\n", { status: 405 }); @@ -151,12 +184,46 @@ } async alarm(): Promise<void> { - if (this.store.getMeta("gc-pending") === "1") { - this.runGc(); - this.store.setMeta("gc-pending", "0"); + if (this.store.getMeta("gc-pending") !== "1") return; + // GC calls packs.reset(): running it while a clone is streaming or a push + // is ingesting corrupts both. Defer past any in-flight upload (leave + // gc-pending set so the re-armed alarm retries) rather than run now. + if (this.activeUploads > 0) { + try { + await this.ctx.storage.setAlarm(Date.now() + 30 * 1000); + } catch { + // alarm store may be busy under a burst; the next push re-arms it + } + return; } + // clear first: at-most-once, so a throwing sweep can't trigger a retry storm + this.store.setMeta("gc-pending", "0"); + try { + await this.gc(); + } catch (err) { + console.log("gc failed", String(err)); + } } + /** + * Run GC serialized against pushes (same receiveChain) and under + * blockConcurrencyWhile, so packs.reset() can never land mid-ingest or + * mid-clone. + */ + private gc(): Promise<{ removed: number; kept: number; skipped?: boolean }> { + let result: { removed: number; kept: number; skipped?: boolean } = { removed: 0, kept: 0 }; + const run = this.receiveChain.then(() => + this.ctx.blockConcurrencyWhile(async () => { + result = this.runGc(); + }) + ); + this.receiveChain = run.then( + () => {}, + () => {} + ); + return run.then(() => result); + } + private handleConfig(body: string): Response { let cfg: Record<string, unknown>; try { @@ -182,19 +249,21 @@ * huge repos skip the sweep entirely — the walk isn't worth it. */ private runGc(): { removed: number; kept: number; skipped?: boolean } { - if (this.store.packs.countObjects() > 300_000) { + if (this.store.objectCount() > 300_000) { return { removed: 0, kept: this.store.objectCount(), skipped: true }; } - const reachable = new Set<string>(); + const reachable = new OidSet(Math.max(this.store.objectCount(), 4096)); const stack: string[] = this.store.refs().map((r) => r.target); const head = this.store.resolveHead(); if (head) stack.push(head); while (stack.length) { const oid = stack.pop()!; - if (reachable.has(oid)) continue; + if (!reachable.addHex(oid)) continue; + const meta = this.store.typeAndSize(oid); + if (!meta) continue; + if (meta.type === "blob") continue; // leaf: never inflate blob content here const obj = this.store.get(oid); if (!obj) continue; - reachable.add(oid); if (obj.type === "commit") { const c = parseCommit(obj.data); stack.push(c.tree, ...c.parents); @@ -207,19 +276,25 @@ } } } - let removed = 0; - for (const oid of this.store.allOids()) { - if (!reachable.has(oid)) { - this.store.deleteObject(oid); - removed++; - } - } - // repack-by-migration: reachable pack objects move to loose storage, - // then the packs (including anything stranded inside them) are dropped + // base_oid is a second reference edge: thin-pack deltas point at bases + // that are frequently unreachable from any ref. Added after the graph walk + // so a base that is itself a reachable tree keeps its own children. + for (const b of this.store.packs.baseOids()) reachable.addHex(b); + const packCount = this.store.packs.countObjects(); - if (packCount > 0) { + // migration duplicates every reachable pack object into loose storage + // before the packs are dropped: guard against crossing the 10GB DO ceiling + // mid-rewrite (which would brick the repo) by leaving the packs intact when + // there is no headroom for the copy. + const noHeadroom = + packCount > 0 && this.store.dbSize() + this.store.packs.totalPackBytes() > DO_STORAGE_CAP; + let removed = 0; + if (packCount > 0 && !noHeadroom) { + // migrate reachable pack objects to loose FIRST, then reset packs, so a + // base or reachable object stranded inside a pack survives the drop let migrated = 0; - for (const oid of reachable) { + for (let i = 0; i < reachable.size; i++) { + const oid = reachable.atHex(i); if (this.store.packs.lookup(oid)) { const obj = this.store.get(oid); if (obj) { @@ -231,12 +306,22 @@ removed += packCount - migrated; this.store.packs.reset(); } - return { removed, kept: reachable.size }; + // sweep loose objects (including anything just migrated) unreachable now + for (const oid of this.store.allOids()) { + if (!reachable.hasHex(oid)) { + this.store.deleteObject(oid); + removed++; + } + } + return noHeadroom ? { removed, kept: reachable.size, skipped: true } : { removed, kept: reachable.size }; } private async readBody(req: Request): Promise<Uint8Array> { let body: Uint8Array = new Uint8Array(await req.arrayBuffer()); - if (req.headers.get("content-encoding")?.includes("gzip")) body = gunzip(body); + if (body.length > MAX_UPLOAD_PACK_BYTES) throw new Error("request body exceeds maximum size"); + if (req.headers.get("content-encoding")?.includes("gzip")) { + body = gunzipLimited(body, MAX_UPLOAD_PACK_BYTES); + } return body; } @@ -248,7 +333,10 @@ * retires its isolate) out from under a long ingest. */ private async receive(req: Request, repo: string): Promise<Response> { - const maxBytes = (parseInt(this.env.MAX_PUSH_MB ?? "", 10) || 8192) * 1024 * 1024; + const maxBytes = (parseInt(this.env.MAX_PUSH_MB ?? "", 10) || DEFAULT_MAX_PUSH_MB) * 1024 * 1024; + // refuse an over-sized push from its declared length: buffering it first + // and checking after is exactly the memory the limit exists to bound + if (overDeclaredLength(req, maxBytes)) return tooLargeResponse(maxBytes); const chunks: Uint8Array[] = []; // chain: a second push (e.g. a client retry of the same POST) must // wait — two interleaved ingests would both claim the next pack id @@ -264,7 +352,9 @@ await run; } catch (err) { console.log(`[receive ${repo}] FAILED: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); - return new Response(`error: ${err instanceof Error ? err.message : String(err)}\n`, { status: 500 }); + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes("exceeds maximum size")) return tooLargeResponse(maxBytes); + return new Response(`error: ${msg}\n`, { status: 500 }); } return new Response(concat(chunks) as unknown as BodyInit, { headers: { @@ -284,7 +374,9 @@ let reader: ReadableStreamDefaultReader<Uint8Array> | null = null; let buf: Uint8Array; if (req.headers.get("content-encoding")?.includes("gzip")) { - buf = gunzip(new Uint8Array(await req.arrayBuffer())); // only small pushes arrive gzipped + // only small pushes arrive gzipped, and the inflate is budgeted: a + // zlib bomb can no more exceed the push cap than a plain body can + buf = gunzipLimited(new Uint8Array(await req.arrayBuffer()), maxBytes); } else { reader = (req.body?.getReader() as ReadableStreamDefaultReader<Uint8Array>) ?? null; buf = new Uint8Array(0); @@ -353,7 +445,11 @@ } } - const { results, changed, needsGc } = applyPushCommands(this.store, commands, unpackError); + // one savepoint around every ref update: a multi-ref push (git push --all) + // must not leave half its branches moved if a later update throws + const { results, changed, needsGc } = this.ctx.storage.transactionSync(() => + applyPushCommands(this.store, commands, unpackError) + ); if (changed) { this.store.setMeta("created", "1"); this.store.setMeta("last-push", String(Date.now())); @@ -428,6 +524,53 @@ } + /** + * Resolve an object id for the browser's object-serving paths (blob/commit/ + * diff/patch/tree-by-oid). Only a full 40-char oid that is reachable from a + * current ref is served: short-prefix resolution let a force-pushed secret be + * brute-forced through the UI even after the protocol side stopped serving + * unreachable objects. + */ + private resolveServable(id: string): string | null { + if (!isOid(id) || !this.store.has(id)) return null; + return this.reachableOid(id) ? id : null; + } + + /** Is `target` reachable from any current ref (or HEAD)? */ + private reachableOid(target: string): boolean { + // the full-clone reachable-set cache is versioned against current refs + // (null on any mismatch), so a hit here is exactly as correct as the walk + // below and skips inflating/parsing the whole object graph + const cached = this.store.loadReachable(); + if (cached) return cached.hasHex(target); + const seen = new OidSet(Math.max(this.store.objectCount(), 4096)); + const stack: string[] = this.store.refs().map((r) => r.target); + const head = this.store.resolveHead(); + if (head) stack.push(head); + while (stack.length) { + const oid = stack.pop()!; + if (oid === target) return true; + if (!seen.addHex(oid)) continue; + const meta = this.store.typeAndSize(oid); + if (!meta) continue; + if (meta.type === "blob") continue; + const obj = this.store.get(oid); + if (!obj) continue; + if (obj.type === "commit") { + const c = parseCommit(obj.data); + stack.push(c.tree, ...c.parents); + } else if (obj.type === "tag") { + const t = parseTag(obj.data); + if (t.object) stack.push(t.object); + } else if (obj.type === "tree") { + for (const e of parseTree(obj.data)) { + if (!isGitlinkMode(e.mode)) stack.push(e.oid); + } + } + } + return false; + } + /** Resolve ?h= (branch, tag, full ref, or oid) to an object id. */ private resolveRef(h?: string): { refName: string | null; oid: string } | null { if (!h) { @@ -438,17 +581,30 @@ const oid = this.store.getRef(cand); if (oid) return { refName: cand, oid }; } - const full = this.store.findOid(h); + const full = this.resolveServable(h); if (full) return { refName: null, oid: full }; return null; } - /** Follow tag objects until we reach a commit. */ + /** + * Follow tag objects until we reach a commit. Tag oids are immutable + * content, so a start oid that actually peeled through a tag has its final + * commit oid cached — a hit skips every intermediate tag object read. + */ private peelToCommit(oid: string): { oid: string; commit: Commit } | null { + const start = oid; + const cached = this.store.getPeeled(start); + if (cached !== null) { + const obj = this.store.get(cached); + return obj?.type === "commit" ? { oid: cached, commit: parseCommit(obj.data) } : null; + } for (let i = 0; i < 10; i++) { const obj = this.store.get(oid); if (!obj) return null; - if (obj.type === "commit") return { oid, commit: parseCommit(obj.data) }; + if (obj.type === "commit") { + if (oid !== start) this.store.setPeeled(start, oid); + return { oid, commit: parseCommit(obj.data) }; + } if (obj.type === "tag") { oid = parseTag(obj.data).object; continue; @@ -476,11 +632,27 @@ return oid; } - private matchesFilter(e: LogEntry, filter: LogFilter): boolean { + /** pathOid keyed by the root tree it starts from: shared across a log walk. */ + private pathOidCached(treeOid: string, path: string[], memo: Map<string, string | null>): string | null { + const hit = memo.get(treeOid); + if (hit !== undefined) return hit; + let oid: string | null = treeOid; + for (const seg of path) { + const obj = this.store.get(oid); + if (obj?.type !== "tree") { oid = null; break; } + const entry = parseTree(obj.data).find((e) => e.name === seg); + if (!entry) { oid = null; break; } + oid = entry.oid; + } + memo.set(treeOid, oid); + return oid; + } + + private matchesFilter(e: LogEntry, filter: LogFilter, memo: Map<string, string | null>): boolean { if (filter.path && filter.path.length) { - const mine = this.pathOid(e.commit, filter.path); - const parent = e.commit.parents[0] ? this.loadCommit(e.commit.parents[0]) : null; - const theirs = parent ? this.pathOid(parent, filter.path) : null; + const mine = this.pathOidCached(e.commit.tree, filter.path, memo); + const parentTree = e.commit.parents[0] ? this.loadCommit(e.commit.parents[0])?.tree : undefined; + const theirs = parentTree ? this.pathOidCached(parentTree, filter.path, memo) : null; if (mine === theirs) return false; } if (filter.q) { @@ -501,14 +673,17 @@ private walkLog(tip: string, skip: number, limit: number, filter: LogFilter = {}): { entries: LogEntry[]; more: boolean } { const first = this.peelToCommit(tip); if (!first) return { entries: [], more: false }; + skip = Math.min(skip, MAX_LOG_SCAN); // a wild ofs must not drive the walk to its cap for an empty page + const scanCap = filter.path?.length ? PATH_LOG_SCAN : MAX_LOG_SCAN; + const memo = new Map<string, string | null>(); const seen = new Set<string>([first.oid]); const frontier: LogEntry[] = [{ oid: first.oid, commit: first.commit }]; const out: LogEntry[] = []; let scanned = 0; - while (frontier.length && out.length < skip + limit + 1 && scanned++ < MAX_LOG_SCAN) { + while (frontier.length && out.length < skip + limit + 1 && scanned++ < scanCap) { frontier.sort((a, b) => b.commit.committer.time - a.commit.committer.time); const cur = frontier.shift()!; - if (this.matchesFilter(cur, filter)) out.push(cur); + if (this.matchesFilter(cur, filter, memo)) out.push(cur); for (const p of cur.commit.parents) { if (seen.has(p)) continue; seen.add(p); @@ -520,10 +695,11 @@ } /** First-parent history of a path (for blame), newest first, with blobs. */ - private pathHistory(tip: string, path: string[], cap: number): BlameHistoryEntry[] { + private pathHistory(tip: string, path: string[], cap: number, maxBytes: number): BlameHistoryEntry[] { const out: BlameHistoryEntry[] = []; let cur = this.peelToCommit(tip); let steps = 0; + let bytes = 0; while (cur && steps++ < MAX_LOG_SCAN && out.length < cap) { const myOid = this.pathOid(cur.commit, path); const parent = cur.commit.parents[0] ? this.peelToCommit(cur.commit.parents[0]) : null; @@ -530,7 +706,12 @@ const parentOid = parent ? this.pathOid(parent.commit, path) : null; if (myOid !== parentOid) { const blob = myOid ? this.store.get(myOid) : null; - out.push({ oid: cur.oid, commit: cur.commit, blob: blob?.type === "blob" ? blob.data : null }); + const data = blob?.type === "blob" ? blob.data : null; + if (data) { + bytes += data.length; + if (bytes > maxBytes && out.length) break; // bound total blobs held in memory + } + out.push({ oid: cur.oid, commit: cur.commit, blob: data }); if (!parentOid) break; // file created here } cur = parent; @@ -1008,7 +1189,15 @@ const rr = this.resolveRef(h); if (!rr) return errorPage(withPath, "empty repository"); if (!path.length) return errorPage(withPath, "blame needs a file path"); - const history = this.pathHistory(rr.oid, path, 200); + // size-gate the tip blob before walking history: a large file would + // otherwise load up to 200 full revisions into memory before blame() bails + const tip = this.peelToCommit(rr.oid); + if (!tip) return errorPage(withPath, "no commit found"); + const tipOid = this.pathOid(tip.commit, path); + const meta = tipOid ? this.store.typeAndSize(tipOid) : null; + if (!meta || meta.type !== "blob") return errorPage(withPath, `no such file: ${path.join("/")}`); + if (meta.size > MAX_BLAME_BYTES) return errorPage(withPath, "blame skipped: file too large"); + const history = this.pathHistory(rr.oid, path, 200, MAX_BLAME_HISTORY_BYTES); if (!history.length || !history[0].blob) return errorPage(withPath, `no such file: ${path.join("/")}`); if (isBinary(history[0].blob)) return errorPage(withPath, "cannot blame a binary file"); const result = blame(history); @@ -1047,7 +1236,7 @@ } private blobByIdPage(id: string): Response { - const oid = this.store.findOid(id); + const oid = this.resolveServable(id); if (!oid) return new Response("not found\n", { status: 404 }); const obj = this.store.get(oid); if (!obj || obj.type !== "blob") return new Response("not a blob\n", { status: 404 }); @@ -1083,7 +1272,7 @@ private resolveCommitId(id: string | undefined, h: string | undefined): string | null { if (id) { - const full = this.store.findOid(id); + const full = this.resolveServable(id); return full ? this.peelToCommit(full)?.oid ?? null : null; } const rr = this.resolveRef(h); @@ -1149,7 +1338,7 @@ const r = `/${encodeURIComponent(repo)}`; const base = this.base(repo, "refs", undefined, `${r}/refs/`); if (!name) return errorPage(base, "no tag given"); - const target = this.store.getRef(`refs/tags/${name}`) ?? this.store.findOid(name); + const target = this.store.getRef(`refs/tags/${name}`) ?? this.resolveServable(name); if (!target) return errorPage(base, `tag not found: ${name}`); const obj = this.store.get(target); if (!obj) return errorPage(base, `missing object`); @@ -1266,13 +1455,18 @@ return new Response(xml, { headers: { "content-type": "application/atom+xml; charset=utf-8" } }); } - private statsPage(repo: string, h: string | undefined, period: string): Response { - const r = `/${encodeURIComponent(repo)}`; - const base = this.base(repo, "stats", h, `${r}/stats/`); - const rr = this.resolveRef(h); - if (!rr) return errorPage(base, "empty repository"); - const { entries } = this.walkLog(rr.oid, 0, MAX_STATS_SCAN); - + /** + * Commit-activity aggregation for a fixed (tip commit oid, period) pair. + * History under an immutable commit oid never changes, so a cache hit + * needs no version check — only the pair itself as key. + */ + private computeStats( + tipOid: string, + period: string + ): { periodKeys: string[]; authors: { name: string; counts: Record<string, number> }[]; totals: Record<string, number>; count: number } { + const cached = this.store.getStatsCache(tipOid, period); + if (cached) return JSON.parse(cached); + const { entries } = this.walkLog(tipOid, 0, MAX_STATS_SCAN); const keyOf = (t: number): string => { const d = new Date(t * 1000); if (period === "y") return String(d.getUTCFullYear()); @@ -1285,20 +1479,41 @@ return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`; }; const periodKeys: string[] = []; - const byAuthor = new Map<string, Map<string, number>>(); - const totals = new Map<string, number>(); + const authorOrder: string[] = []; + const byAuthor = new Map<string, Record<string, number>>(); + const totals: Record<string, number> = {}; for (const e of entries) { const key = keyOf(e.commit.committer.time); if (!periodKeys.includes(key)) periodKeys.push(key); const author = e.commit.author.name || "(unknown)"; - if (!byAuthor.has(author)) byAuthor.set(author, new Map()); + if (!byAuthor.has(author)) { + byAuthor.set(author, {}); + authorOrder.push(author); + } const m = byAuthor.get(author)!; - m.set(key, (m.get(key) ?? 0) + 1); - totals.set(key, (totals.get(key) ?? 0) + 1); + m[key] = (m[key] ?? 0) + 1; + totals[key] = (totals[key] ?? 0) + 1; } - const cols = periodKeys.slice(0, 8); - const authors = [...byAuthor.entries()] - .map(([name, m]) => ({ name, m, total: [...m.values()].reduce((a, b) => a + b, 0) })) + const result = { + periodKeys, + authors: authorOrder.map((name) => ({ name, counts: byAuthor.get(name)! })), + totals, + count: entries.length, + }; + this.store.setStatsCache(tipOid, period, JSON.stringify(result)); + return result; + } + + private statsPage(repo: string, h: string | undefined, period: string): Response { + const r = `/${encodeURIComponent(repo)}`; + const base = this.base(repo, "stats", h, `${r}/stats/`); + const rr = this.resolveRef(h); + if (!rr) return errorPage(base, "empty repository"); + const agg = this.computeStats(rr.oid, period); + + const cols = agg.periodKeys.slice(0, 8); + const authors = agg.authors + .map((a) => ({ name: a.name, m: a.counts, total: Object.values(a.counts).reduce((x, y) => x + y, 0) })) .sort((a, b) => b.total - a.total) .slice(0, 20); const periodLinks = [ @@ -1317,16 +1532,16 @@ .map( (a) => `<tr><td class='name'>${esc(a.name)}</td>` + - cols.map((k) => `<td>${a.m.get(k) ?? ""}</td>`).join("") + + cols.map((k) => `<td>${a.m[k] ?? ""}</td>`).join("") + `<td><strong>${a.total}</strong></td></tr>` ) .join(""); const totalRow = `<tr><td class='name'><strong>Total</strong></td>` + - cols.map((k) => `<td><strong>${totals.get(k) ?? 0}</strong></td>`).join("") + - `<td><strong>${entries.length}</strong></td></tr>`; + cols.map((k) => `<td><strong>${agg.totals[k] ?? 0}</strong></td>`).join("") + + `<td><strong>${agg.count}</strong></td></tr>`; const body = ` -<div>Commits per author per ${period === "w" ? "week" : period === "y" ? "year" : period === "q" ? "quarter" : "month"} (${periodLinks})${entries.length >= MAX_STATS_SCAN ? ` — last ${MAX_STATS_SCAN} commits` : ""}</div> +<div>Commits per author per ${period === "w" ? "week" : period === "y" ? "year" : period === "q" ? "quarter" : "month"} (${periodLinks})${agg.count >= MAX_STATS_SCAN ? ` — last ${MAX_STATS_SCAN} commits` : ""}</div> <table class='stats'> <tr><th>Author</th>${cols.map((k) => `<th>${esc(k)}</th>`).join("")}<th>Total</th></tr> ${rows} @@ -1336,6 +1551,25 @@ } } +/** + * Reject an oversized body from its declared length, before a byte of it is + * read. content-length is advisory (chunked and gzipped bodies under-report or + * omit it), so this is the cheap first gate only — the in-stream ingest cap and + * the bounded gunzip still have to hold for everything that gets past it. + */ +function overDeclaredLength(req: Request, maxBytes: number): boolean { + const declared = parseInt(req.headers.get("content-length") ?? "", 10); + return Number.isFinite(declared) && declared > maxBytes; +} + +function tooLargeResponse(maxBytes: number): Response { + const mb = Math.round(maxBytes / (1024 * 1024)); + return new Response(`error: request body exceeds the ${mb} MiB limit for this server\n`, { + status: 413, + headers: { "content-type": "text/plain; charset=utf-8" }, + }); +} + function renderHunk(hk: Hunk): string { let html = `<div class='hunk'>@@ -${hk.aStart},${hk.aLen} +${hk.bStart},${hk.bLen} @@</div>`; for (const op of hk.ops) { @@ -1350,10 +1584,16 @@ const ext = name.slice(name.lastIndexOf(".") + 1).toLowerCase(); const types: Record<string, string> = { png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif", - svg: "image/svg+xml", pdf: "application/pdf", html: "text/plain", + svg: "text/plain; charset=utf-8", pdf: "application/pdf", html: "text/plain", }; const ct = types[ext] ?? (isBinary(data) ? "application/octet-stream" : "text/plain; charset=utf-8"); - return new Response(data as unknown as BodyInit, { headers: { "content-type": ct } }); + return new Response(data as unknown as BodyInit, { + headers: { + "content-type": ct, + "x-content-type-options": "nosniff", + "content-disposition": "inline", + }, + }); } function decodePath(p: string): string[] { diff --git a/src/ui/html.ts b/src/ui/html.ts @@ -1,9 +1,9 @@ +const ESCAPES: Record<string, string> = { + "&": "&", "<": "<", ">": ">", '"': """, "'": "'", "`": "`", +}; + export function esc(s: string): string { - return s - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """); + return s.replace(/[&<>"'`]/g, (c) => ESCAPES[c]); } /** cgit-style relative age with its color class, e.g. "3 days" / age-days. */ @@ -88,7 +88,7 @@ const switcher = o.repo && o.branches && o.branches.length ? `<td class='form'><form method='get' action='${esc(o.formAction ?? `${r}/`)}'>` + - `<select name='h' onchange='this.form.submit();'>` + + `<select name='h'>` + o.branches .map((b) => `<option value='${esc(b)}'${b === o.ref ? " selected='selected'" : ""}>${esc(b)}</option>`) .join("") + @@ -126,10 +126,19 @@ `; } +const CSP = + "default-src 'none'; img-src 'self' https: data:; style-src 'self' 'unsafe-inline'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'"; + export function htmlResponse(html: string, status = 200): Response { return new Response(html, { status, - headers: { "content-type": "text/html; charset=utf-8" }, + headers: { + "content-type": "text/html; charset=utf-8", + "content-security-policy": CSP, + "x-content-type-options": "nosniff", + "referrer-policy": "no-referrer", + "x-frame-options": "DENY", + }, }); } |