Measuring your real landing rate

1/**
2 * Record the conditions of every submission, at the moment you submit.
3 *
4 * This is the whole discipline. A landing rate with no context is a number you
5 * can watch and cannot act on. The same number split by blockhash freshness,
6 * fee percentile and venue tells you which lever to pull.
7 *
8 * Record it synchronously, before the send resolves. If you record after, you
9 * will silently drop the submissions that threw.
10 */
11export type Submission = {
12  signature: string;
13  sentAtMs: number;
14 
15  // Conditions you control, and therefore can correlate against.
16  blockhashSlotsRemaining: number;
17  feeMicroLamports: number;
18  requestedComputeUnits: number;
19  transport: "http" | "quic";
20  route: string;                  // venue, program, or strategy name
21  writableAccounts: string[];
22 
23  // Filled in later by reconciliation.
24  outcome?: "landed_ok" | "landed_failed" | "never_seen";
25  landedSlot?: number;
26  onChainError?: unknown;
27  resolvedAtMs?: number;
28};
29 
30export class SubmissionLog {
31  private open = new Map<string, Submission>();
32  private closed: Submission[] = [];
33 
34  record(s: Submission) {
35    this.open.set(s.signature, s);
36  }
37 
38  resolve(signature: string, patch: Partial<Submission>) {
39    const s = this.open.get(signature);
40    if (!s) return;
41    Object.assign(s, patch, { resolvedAtMs: Date.now() });
42    this.open.delete(signature);
43    this.closed.push(s);
44  }
45 
46  /** Anything still open past the blockhash window can never land. */
47  expire(olderThanMs = 90_000) {
48    const cutoff = Date.now() - olderThanMs;
49    for (const [sig, s] of this.open) {
50      if (s.sentAtMs < cutoff) {
51        this.resolve(sig, { outcome: "never_seen" });
52      }
53    }
54  }
55 
56  drain(): Submission[] {
57    const out = this.closed;
58    this.closed = [];
59    return out;
60  }
61 
62  get pending() {
63    return [...this.open.values()];
64  }
65}

1import { Connection } from "@solana/web3.js";
2import type { SubmissionLog } from "./log";
3 
4/**
5 * Ask the chain what happened, in batches, on a loop.
6 *
7 * Two details decide whether this is truthful:
8 *
9 *   searchTransactionHistory  without it you only see the recent status cache,
10 *                             a few hundred slots deep, and everything older
11 *                             comes back null. This single flag is the most
12 *                             common cause of a landing rate that looks far
13 *                             worse than reality.
14 *
15 *   batching                  getSignatureStatuses takes up to 256 signatures.
16 *                             One call per signature will rate limit you and
17 *                             the gaps will look like failures.
18 */
19export async function reconcile(rpc: Connection, log: SubmissionLog) {
20  const pending = log.pending;
21  if (pending.length === 0) return;
22 
23  for (let i = 0; i < pending.length; i += 256) {
24    const batch = pending.slice(i, i + 256);
25    const { value } = await rpc.getSignatureStatuses(
26      batch.map((s) => s.signature),
27      { searchTransactionHistory: true },
28    );
29 
30    batch.forEach((submission, j) => {
31      const status = value[j];
32      if (!status) return;                     // not seen yet, leave it open
33 
34      if (status.err) {
35        log.resolve(submission.signature, {
36          outcome: "landed_failed",
37          landedSlot: status.slot,
38          onChainError: status.err,
39        });
40      } else if (status.confirmationStatus) {
41        log.resolve(submission.signature, {
42          outcome: "landed_ok",
43          landedSlot: status.slot,
44        });
45      }
46    });
47  }
48 
49  // Anything still open past the window is provably dead. Closing these is
50  // what stops your "pending" bucket growing forever and quietly inflating
51  // your apparent success rate.
52  log.expire();
53}

1import type { Submission } from "./log";
2 
3/**
4 * Landing rate, sliced by the things you control.
5 *
6 * An aggregate number tells you whether you have a problem. These slices tell
7 * you which one, and that is the difference between a metric and a diagnosis.
8 */
9export function analyse(submissions: Submission[]) {
10  const bucket = <K extends string>(key: (s: Submission) => K) => {
11    const rows = new Map<K, { total: number; ok: number; failed: number; missing: number }>();
12    for (const s of submissions) {
13      const k = key(s);
14      const row = rows.get(k) ?? { total: 0, ok: 0, failed: 0, missing: 0 };
15      row.total += 1;
16      if (s.outcome === "landed_ok") row.ok += 1;
17      else if (s.outcome === "landed_failed") row.failed += 1;
18      else row.missing += 1;
19      rows.set(k, row);
20    }
21    return [...rows.entries()].map(([k, r]) => ({
22      key: k,
23      total: r.total,
24      // Delivery rate: did it reach a block at all, regardless of outcome.
25      deliveryRate: (r.ok + r.failed) / r.total,
26      // Success rate: did it reach a block AND do what you wanted.
27      successRate: r.ok / r.total,
28      neverSeenRate: r.missing / r.total,
29    }));
30  };
31 
32  return {
33    byBlockhashFreshness: bucket((s) =>
34      s.blockhashSlotsRemaining > 120 ? "fresh"
35      : s.blockhashSlotsRemaining > 60 ? "middling"
36      : "stale",
37    ),
38    byFeeDecile: bucket((s) => `d${Math.min(9, Math.floor(Math.log10(Math.max(1, s.feeMicroLamports))))}`),
39    byTransport: bucket((s) => s.transport),
40    byRoute: bucket((s) => s.route),
41    byHour: bucket((s) => String(new Date(s.sentAtMs).getUTCHours())),
42  };
43}
44 
45/**
46 * The reading:
47 *
48 *   deliveryRate varies with blockhash freshness  -> window management
49 *   deliveryRate varies with fee decile           -> underpricing
50 *   deliveryRate flat across everything           -> the path
51 *   successRate low but deliveryRate high         -> not a delivery problem
52 *                                                    at all, look at slippage
53 *                                                    and compute
54 */