scripts/sepolia/probe-rpcs.ts285 lines · 16642 bytes
| 1 | #!/usr/bin/env bun |
| 2 | // Read-only provider check for the CRE clearance workflow: runs the workflow's own judging code against each RPC |
| 3 | // provider (cre/veto/judge.ts judgeOnProvider: eth_getCode, the attack simulation in 1 to simRequests eth_simulateV1 |
| 4 | // requests, R9 on the fetched code; the JSON-RPC bodies come from workflow.ts httpRpcBatch), then the workflow's |
| 5 | // quorum, with curl as the transport instead of the CRE HTTP capability, and times every request. The static rules |
| 6 | // (R5, R8) and approvals are not read: every case is judged as an unapproved upgrade without init data. The test |
| 7 | // holder and the holder that funds the stranger are picked as the workflow picks them. |
| 8 | // |
| 9 | // bun scripts/sepolia/probe-rpcs.ts # the live deployment (deployments/sepolia.json): |
| 10 | // # evilMint/evilOwner/evilFee/evilBaseFee/evilDrain/v2 |
| 11 | // # must give R1/R2/R3/R9/R3B/pass |
| 12 | // bun scripts/sepolia/probe-rpcs.ts --proxy <erc1967 proxy> --impl <implementation> --holder <address>[,<address>] |
| 13 | // # before the deployment: any ERC-1967 token proxy; the |
| 14 | // # second holder, if any, funds the stranger |
| 15 | // Options: --deployment file, --config file (providers and tuning; default cre/veto/config.sepolia.json, else the |
| 16 | // built-in four), --rpc label=url (repeat; replaces the providers), --repeat n, --max-calls n, --block n, |
| 17 | // --json file (raw measurements). FORK=<port> picks the fork deployment and points three providers at it. |
| 18 | // Nothing is sent: eth_getCode, eth_call, eth_simulateV1 and block reads only. |
| 19 | // Exit status 0 when every quorum verdict is the expected one (with a deployment), every request fits the CRE size |
| 20 | // limits and the providers fit the 15 HTTP calls of one run; providers that fail are listed but do not fail the |
| 21 | // probe while the quorum holds. |
| 22 | import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' |
| 23 | import { tmpdir } from 'node:os' |
| 24 | import { join, resolve } from 'node:path' |
| 25 | |
| 26 | const ROOT = resolve(import.meta.dir, '..', '..') |
| 27 | const VETO = resolve(ROOT, 'cre/veto') |
| 28 | const { judgeOnProvider, quorum, consensusValue } = await import(resolve(VETO, 'judge.ts')) |
| 29 | const { httpRpcBatch } = await import(resolve(VETO, 'workflow.ts')) |
| 30 | const { proposalActors } = await import(resolve(VETO, 'simulate.ts')) |
| 31 | const { stableStringify } = await import(resolve(VETO, 'report.ts')) |
| 32 | const { keccak256 } = await import(Bun.resolveSync('viem', VETO)) |
| 33 | |
| 34 | // cre workflow limits export (CRE CLI v1.37.0): HTTPAction and Consensus |
| 35 | const LIMITS = { httpCalls: 15, requestBytes: 120_000, responseBytes: 250_000, timeoutMs: 10_000, observationBytes: 25_000 } |
| 36 | const DEFAULT_RPCS = [ |
| 37 | { url: 'https://ethereum-sepolia-rpc.publicnode.com', label: 'publicnode' }, |
| 38 | { url: 'https://sepolia.drpc.org', label: 'drpc' }, |
| 39 | { url: 'https://11155111.rpc.thirdweb.com', label: 'thirdweb' }, |
| 40 | { url: 'https://1rpc.io/sepolia', label: '1rpc' }, |
| 41 | ] |
| 42 | const EXPECT: Record<string, string> = { |
| 43 | evilMint: 'R1_TOTAL_SUPPLY', |
| 44 | evilOwner: 'R2_OWNER', |
| 45 | evilFee: 'R3_TRANSFER_CONSERVATION', |
| 46 | evilBaseFee: 'R9_NEW_OPCODES', |
| 47 | evilDrain: 'R3B_THIRD_PARTY_FUNDS', |
| 48 | v2: 'pass', |
| 49 | } |
| 50 | |
| 51 | type Rpc = { url: string; label: string } |
| 52 | type Metric = { provider: string; case: string; round: number; batch: string; http: number; ms: number; ttfbMs: number; reqBytes: number; respBytes: number; simCalls?: number; override?: string; error?: string } |
| 53 | |
| 54 | const die = (m: string): never => { |
| 55 | console.error(`probe-rpcs: ${m}`) |
| 56 | process.exit(2) |
| 57 | } |
| 58 | |
| 59 | // ---------------------------------------------------------------- args |
| 60 | const argv = process.argv.slice(2) |
| 61 | const opt: Record<string, string> = {} |
| 62 | const rpcArgs: Rpc[] = [] |
| 63 | for (let i = 0; i < argv.length; i++) { |
| 64 | const a = argv[i] |
| 65 | if (!a.startsWith('--')) die(`unexpected argument ${a}`) |
| 66 | const v = argv[++i] ?? die(`${a} needs a value`) |
| 67 | if (a === '--rpc') { |
| 68 | const eq = v.indexOf('=') |
| 69 | if (eq < 1) die('--rpc wants label=url') |
| 70 | rpcArgs.push({ label: v.slice(0, eq), url: v.slice(eq + 1) }) |
| 71 | } else opt[a.slice(2)] = v |
| 72 | } |
| 73 | const fork = process.env.FORK |
| 74 | const tmpBase = (process.env.TMPDIR ?? '/tmp').replace(/\/$/, '') |
| 75 | const forkState = fork ? join(tmpBase, `upgrade-airlock-fork-${fork}`) : null |
| 76 | const repeat = Number(opt.repeat ?? 1) |
| 77 | |
| 78 | let config: any = null |
| 79 | const cfgPath = opt.config ?? resolve(VETO, 'config.sepolia.json') |
| 80 | if (opt.config || (!fork && existsSync(cfgPath))) config = JSON.parse(readFileSync(cfgPath, 'utf8')) |
| 81 | let rpcs: Rpc[] = rpcArgs.length ? rpcArgs : config?.rpcs ?? DEFAULT_RPCS |
| 82 | if (fork && !rpcArgs.length) rpcs = ['fork-a', 'fork-b', 'fork-c'].map((label) => ({ label, url: `http://127.0.0.1:${fork}` })) |
| 83 | const minAgree = Number(config?.minAgree ?? 2) |
| 84 | const tuning = { |
| 85 | callGas: config?.callGas ?? 1_000_000, |
| 86 | readGas: config?.readGas ?? 200_000, |
| 87 | maxCalls: Number(opt['max-calls'] ?? config?.maxCalls ?? 150), |
| 88 | simRequests: Number(config?.simRequests ?? 2), |
| 89 | allowedOpcodes: (config?.allowedOpcodes ?? []) as string[], |
| 90 | } |
| 91 | |
| 92 | // ---------------------------------------------------------------- transport (curl, synchronous like the CRE capability) |
| 93 | const work = mkdtempSync(join(tmpBase, 'airlock-probe-')) |
| 94 | process.on('exit', () => rmSync(work, { recursive: true, force: true })) |
| 95 | const metrics: Metric[] = [] |
| 96 | let current = { provider: '', case: '', round: 0 } |
| 97 | |
| 98 | function curl(url: string, body: Uint8Array) { |
| 99 | const out = join(work, 'resp') |
| 100 | const p = Bun.spawnSync(['curl', '-sS', '-m', '30', '-o', out, '-w', '%{http_code} %{time_total} %{time_starttransfer}', '-H', 'content-type: application/json', '--data-binary', '@-', url], { stdin: body }) |
| 101 | const [code, total, ttfb] = new TextDecoder().decode(p.stdout).trim().split(' ') |
| 102 | const resp = existsSync(out) ? readFileSync(out) : new Uint8Array() |
| 103 | rmSync(out, { force: true }) |
| 104 | return { statusCode: Number(code) || 0, body: new Uint8Array(resp), ms: Math.round(Number(total) * 1000), ttfbMs: Math.round(Number(ttfb) * 1000), err: p.exitCode ? new TextDecoder().decode(p.stderr).trim() : '' } |
| 105 | } |
| 106 | |
| 107 | const batchOk = (body: Uint8Array, n: number) => { |
| 108 | try { |
| 109 | const j = JSON.parse(new TextDecoder().decode(body)) |
| 110 | return Array.isArray(j) && j.length === n && j.every((x) => x.result !== undefined) |
| 111 | } catch { |
| 112 | return false |
| 113 | } |
| 114 | } |
| 115 | const timedSend = (req: { url: string; method: string; headers: Record<string, string>; body: string }) => { |
| 116 | const body = Buffer.from(req.body, 'base64') |
| 117 | const calls = JSON.parse(body.toString('utf8')) as { method: string; params: any[] }[] |
| 118 | const r = curl(req.url, body) |
| 119 | const sim = calls.find((c) => c.method === 'eth_simulateV1') |
| 120 | const ovr = sim ? Object.values(sim.params[0].blockStateCalls[0].stateOverrides ?? {}).map((o: any) => Object.keys(o).join('+')).join(',') : undefined |
| 121 | metrics.push({ |
| 122 | ...current, |
| 123 | batch: calls.map((c) => c.method.replace(/^eth_/, '')).join('+'), |
| 124 | http: r.statusCode, |
| 125 | ms: r.ms, |
| 126 | ttfbMs: r.ttfbMs, |
| 127 | reqBytes: body.length, |
| 128 | respBytes: r.body.length, |
| 129 | simCalls: sim ? sim.params[0].blockStateCalls[0].calls.length : undefined, |
| 130 | override: ovr, |
| 131 | error: r.err || (r.statusCode !== 200 || !batchOk(r.body, calls.length) ? new TextDecoder().decode(r.body).slice(0, 160) : undefined), |
| 132 | }) |
| 133 | return { statusCode: r.statusCode, body: r.body } |
| 134 | } |
| 135 | const rpcBatch = httpRpcBatch(timedSend) |
| 136 | |
| 137 | // plain single calls for setup (not measured) |
| 138 | function call(url: string, method: string, params: unknown[]): any { |
| 139 | const r = curl(url, new TextEncoder().encode(JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }))) |
| 140 | const j = JSON.parse(new TextDecoder().decode(r.body) || '{}') |
| 141 | if (j.error || j.result === undefined) die(`${method} on ${url}: ${j.error?.message ?? r.err ?? 'no result'}`) |
| 142 | return j.result |
| 143 | } |
| 144 | const balanceOf = (url: string, token: string, who: string, tag: string) => |
| 145 | BigInt(call(url, 'eth_call', [{ to: token, data: `0x70a08231${who.slice(2).toLowerCase().padStart(64, '0')}` }, tag])) |
| 146 | |
| 147 | // ---------------------------------------------------------------- what to judge |
| 148 | type Case = { name: string; impl: string; expect?: string } |
| 149 | let proxy: string |
| 150 | let holders: string[] |
| 151 | let watch: string[] |
| 152 | let actorAnchor: string |
| 153 | let cases: Case[] |
| 154 | const depPath = opt.deployment ?? (forkState ? join(forkState, 'deployment.json') : resolve(ROOT, 'deployments/sepolia.json')) |
| 155 | if (opt.proxy) { |
| 156 | if (!opt.impl || !opt.holder) die('--proxy needs --impl and --holder') |
| 157 | proxy = opt.proxy |
| 158 | holders = opt.holder.split(',').map((h) => h.trim()).filter(Boolean) |
| 159 | watch = [] |
| 160 | actorAnchor = opt.proxy |
| 161 | cases = [{ name: 'impl', impl: opt.impl }] |
| 162 | } else { |
| 163 | if (!existsSync(depPath)) die(`no deployment at ${depPath}; use --proxy/--impl/--holder before the deployment`) |
| 164 | const d = JSON.parse(readFileSync(depPath, 'utf8')) |
| 165 | if (!d.airlockAdmin) die(`${depPath} has no airlockAdmin, so it is not a deployment of this stack (deploy it with deploy.sh)`) |
| 166 | proxy = d.proxy |
| 167 | holders = d.holders |
| 168 | watch = config?.watch ?? [d.owner, d.airlockAdmin] |
| 169 | actorAnchor = d.airlockAdmin |
| 170 | cases = Object.keys(EXPECT) |
| 171 | .filter((k) => d[k]) |
| 172 | .map((k) => ({ name: k, impl: d[k], expect: EXPECT[k] })) |
| 173 | } |
| 174 | |
| 175 | const ref = rpcs[0].url |
| 176 | const head = BigInt(call(ref, 'eth_blockNumber', [])) |
| 177 | const block = opt.block ? BigInt(opt.block) : head - 2n // a block every provider already has |
| 178 | const tag = `0x${block.toString(16)}` |
| 179 | const blockHash = call(ref, 'eth_getBlockByNumber', [tag, false]).hash as `0x${string}` |
| 180 | // As the workflow does (cre/veto/airlock.ts judgeProposal): the first holder with a balance makes the test transfer, |
| 181 | // the next one with a balance funds the stranger with the test amount (capped at its balance). |
| 182 | const balances = holders.map((h) => balanceOf(ref, proxy, h, tag)) |
| 183 | const hk = balances.findIndex((b) => b > 0n) |
| 184 | if (hk < 0) die('no holder with a balance at the probe block') |
| 185 | const holder = holders[hk].toLowerCase() |
| 186 | const bal = balances[hk] |
| 187 | const amount = bal / 100n > 0n ? bal / 100n : 1n |
| 188 | const fk = balances.findIndex((b, i) => b > 0n && i !== hk && holders[i].toLowerCase() !== holder) |
| 189 | const funder = fk >= 0 ? holders[fk].toLowerCase() : null |
| 190 | const fundAmount = fk >= 0 ? (balances[fk] < amount ? balances[fk] : amount) : 0n |
| 191 | const { stranger, receiver } = proposalActors(11155111n, actorAnchor.toLowerCase(), 0n, blockHash) |
| 192 | console.log(`probe block ${block} (head ${head}); proxy ${proxy}; holder ${holder}; stranger funded by ${funder ?? 'nobody (one funded holder)'}; providers ${rpcs.map((r) => r.label).join(', ')}; ${repeat} round(s)`) |
| 193 | |
| 194 | // ---------------------------------------------------------------- run |
| 195 | // Blocking: a quorum verdict that is not the expected one, or a batch over a CRE size limit (identical on every |
| 196 | // provider). Not blocking: a single provider failing, as long as the quorum still holds (it costs time, not a verdict). |
| 197 | let blocking = 0 |
| 198 | const providerFails = new Map<string, number>() |
| 199 | const rows: string[] = [] |
| 200 | for (let round = 1; round <= repeat; round++) { |
| 201 | for (const c of cases) { |
| 202 | const code = call(ref, 'eth_getCode', [c.impl, tag]) as `0x${string}` |
| 203 | const ctx = { |
| 204 | proxy: proxy.toLowerCase(), |
| 205 | newImpl: c.impl.toLowerCase(), |
| 206 | codehash: keccak256(code), |
| 207 | hasInitData: false, |
| 208 | block, |
| 209 | holder, |
| 210 | amount, |
| 211 | funder, |
| 212 | fundAmount, |
| 213 | holders: holders.map((a) => a.toLowerCase()), |
| 214 | watch: watch.map((a) => a.toLowerCase()), |
| 215 | stranger, |
| 216 | receiver, |
| 217 | strict: false, |
| 218 | codehashApproved: false, |
| 219 | dataApproved: false, |
| 220 | ...tuning, |
| 221 | } |
| 222 | const outcomes = [] |
| 223 | for (const p of rpcs) { |
| 224 | current = { provider: p.label, case: c.name, round } |
| 225 | const t0 = performance.now() |
| 226 | const o = judgeOnProvider(rpcBatch, p.url, p.label, ctx) |
| 227 | const wall = Math.round(performance.now() - t0) |
| 228 | outcomes.push(o) |
| 229 | const got = o.verdict === 'error' ? `error: ${o.error.slice(0, 90)}` : o.verdict === 'pass' ? 'pass' : (o.rule as string) |
| 230 | // without an expectation (a foreign proxy) any answer counts, except R6 from a simulation that never ran |
| 231 | const ok = o.verdict !== 'error' && (c.expect ? got === c.expect : got !== 'R6_SIM_FAILURE') |
| 232 | if (!ok) providerFails.set(p.label, (providerFails.get(p.label) ?? 0) + 1) |
| 233 | const why = o.verdict !== 'error' && o.rule === 'R6_SIM_FAILURE' ? ` [${o.checks.find((k: any) => !k.ok)?.after ?? ''}]` : '' |
| 234 | const ms = metrics.filter((m) => m.provider === p.label && m.case === c.name && m.round === round) |
| 235 | rows.push(`${ok ? 'ok ' : 'FAIL'} r${round} ${c.name.padEnd(11)} ${p.label.padEnd(10)} ${got.padEnd(26)} ${ms.map((m) => `${m.batch} ${m.ms}ms`).join(' | ')} (judge ${wall}ms)${why}`) |
| 236 | } |
| 237 | const v = quorum(outcomes, minAgree) |
| 238 | const obs = stableStringify(consensusValue(v)).length |
| 239 | const qGot = v.verdict === 'pass' ? 'pass' : (v.rule as string) |
| 240 | const qOk = (c.expect ? qGot === c.expect : qGot !== 'R6_SIM_FAILURE') && obs <= LIMITS.observationBytes |
| 241 | if (!qOk) blocking++ |
| 242 | const answered = outcomes.filter((o: any) => o.verdict !== 'error').length |
| 243 | rows.push(`${qOk ? 'ok ' : 'FAIL'} r${round} ${c.name.padEnd(11)} QUORUM ${qGot.padEnd(26)} ${answered}/${outcomes.length} providers answered (min ${minAgree}); consensus observation ${obs} bytes`) |
| 244 | } |
| 245 | } |
| 246 | for (const r of rows) console.log(r) |
| 247 | |
| 248 | // ---------------------------------------------------------------- summary per provider and batch |
| 249 | const pct = (xs: number[], q: number) => xs.sort((a, b) => a - b)[Math.min(xs.length - 1, Math.floor(q * xs.length))] |
| 250 | console.log('\nprovider batch n min/median/max ms ttfb med max req B max resp B sim calls override') |
| 251 | for (const p of rpcs) { |
| 252 | for (const b of [...new Set(metrics.filter((m) => m.provider === p.label).map((m) => m.batch))]) { |
| 253 | const ms = metrics.filter((m) => m.provider === p.label && m.batch === b) |
| 254 | const t = ms.map((m) => m.ms) |
| 255 | const req = Math.max(...ms.map((m) => m.reqBytes)) |
| 256 | const resp = Math.max(...ms.map((m) => m.respBytes)) |
| 257 | const sim = [...new Set(ms.map((m) => m.simCalls).filter((x) => x !== undefined))].join('/') |
| 258 | const errs = ms.filter((m) => m.error || m.http !== 200) |
| 259 | const sizes = [req > LIMITS.requestBytes && 'REQUEST>120kb', resp > LIMITS.responseBytes && 'RESPONSE>250kb'].filter(Boolean) |
| 260 | if (sizes.length) blocking++ |
| 261 | const slow = ms.filter((m) => m.ms > LIMITS.timeoutMs).length |
| 262 | if (slow) providerFails.set(p.label, (providerFails.get(p.label) ?? 0) + slow) |
| 263 | const over = [...sizes, slow && `${slow} slower than 10 s`, errs.length && `${errs.length} failed: ${errs[0].error?.replace(/\s+/g, ' ').slice(0, 90)}`].filter(Boolean) |
| 264 | console.log( |
| 265 | `${p.label.padEnd(10)} ${b.padEnd(26)} ${String(ms.length).padStart(2)} ${`${Math.min(...t)}/${pct([...t], 0.5)}/${Math.max(...t)}`.padEnd(18)} ${String(pct(ms.map((m) => m.ttfbMs), 0.5)).padStart(8)} ${String(req).padStart(9)} ${String(resp).padStart(10)} ${(sim || '-').padStart(9)} ${ms[0].override ?? '-'}${over.length ? ` !! ${over.join(', ')}` : ''}`, |
| 266 | ) |
| 267 | } |
| 268 | } |
| 269 | // HTTP calls of one upgrade judgement: per provider eth_getCode, then 1 to simRequests eth_simulateV1 requests (as |
| 270 | // many as the probes need). The workflow refuses a config whose worst case exceeds the per-run limit. |
| 271 | const perProposalMax = (1 + tuning.simRequests) * rpcs.length |
| 272 | const perCase = new Map<string, number>() |
| 273 | for (const m of metrics) perCase.set(`${m.case}#${m.round}`, (perCase.get(`${m.case}#${m.round}`) ?? 0) + 1) |
| 274 | const perProposalUsed = Math.max(0, ...perCase.values()) |
| 275 | if (perProposalMax > LIMITS.httpCalls) blocking++ |
| 276 | console.log( |
| 277 | `\nHTTP calls per upgrade: at most ${perProposalMax} (${rpcs.length} providers x (1 + ${tuning.simRequests})), ${perProposalUsed} in the largest case here; CRE limit ${LIMITS.httpCalls} per run${perProposalMax > LIMITS.httpCalls ? ' !! over the limit' : ''}`, |
| 278 | ) |
| 279 | console.log(`request <= ${LIMITS.requestBytes} B, response <= ${LIMITS.responseBytes} B, ${LIMITS.timeoutMs / 1000} s per call`) |
| 280 | if (opt.json) writeFileSync(opt.json, JSON.stringify({ block: block.toString(), rpcs, metrics }, null, 2)) |
| 281 | const flaky = [...providerFails.entries()].map(([l, n]) => `${l} (${n} of ${repeat * cases.length} runs)`).join(', ') |
| 282 | if (blocking) console.log(`PROBE: FAIL, ${blocking} blocking problem(s)${flaky ? `; failing providers: ${flaky}` : ''}`) |
| 283 | else if (flaky) console.log(`PROBE: QUORUM OK, every verdict as expected; failing providers: ${flaky}`) |
| 284 | else console.log('PROBE: ALL OK') |
| 285 | process.exit(blocking ? 1 : 0) |