From 9a48258e023c9a5772676c7597bdbe6737833f9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20Raz=20Guzm=C3=A1n=20Macedo?= <miguel@zed.dev>
Date: Fri, 29 May 2026 00:37:50 -0600
Subject: [PATCH] Add script to compare criterion benchmarks between branches

This script automates benchmarking a crate on two branches and comparing
the results. It checks out each branch, runs the specified benchmark
with consistent settings, and uses critcmp (or criterion's built-in
comparison) to display differences. The script restores the original
branch on exit and refuses to run with uncommitted changes.
---
 script/compare-bench | 64 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100755 script/compare-bench

diff --git a/script/compare-bench b/script/compare-bench
new file mode 100755
index 00000000000..b04add53adc
--- /dev/null
+++ b/script/compare-bench
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Compare a criterion benchmark between two git branches.
+#
+# Usage:
+#   script/compare-bench CRATE [BASE_BRANCH] [FEATURE_BRANCH]
+#
+# CRATE is the cargo package to benchmark (e.g. `streaming_diff`); by default
+# the bench target of the same name is run.
+#
+# The bench target name can be overridden via the BENCH_NAME environment
+# variable. The `--save-baseline <name>` flag is appended automatically.
+
+if [[ $# -lt 1 ]]; then
+    echo "usage: script/compare-bench CRATE [BASE_BRANCH] [FEATURE_BRANCH]" >&2
+    exit 1
+fi
+
+CRATE="$1"
+BASE_BRANCH="${2:-main}"
+FEATURE_BRANCH="${3:-streaming-edits-f32}"
+BENCH_NAME="${BENCH_NAME:-$CRATE}"
+
+BENCH_CMD=(
+    cargo bench -p "$CRATE" --bench "$BENCH_NAME" --profile release-fast --
+    --warm-up-time 1 --measurement-time 5
+)
+
+run_bench() {
+    local baseline_name="$1"
+    "${BENCH_CMD[@]}" --save-baseline "$baseline_name"
+}
+
+# Refuse to run with uncommitted changes, since we switch branches.
+if ! git diff --quiet || ! git diff --cached --quiet; then
+    echo "error: working tree has uncommitted changes; commit or stash first" >&2
+    exit 1
+fi
+
+# Remember where we started so we can restore it on exit.
+ORIGINAL_REF="$(git symbolic-ref --quiet --short HEAD || git rev-parse HEAD)"
+restore() {
+    git checkout --quiet "$ORIGINAL_REF"
+}
+trap restore EXIT
+
+echo "==> Benchmarking $BASE_BRANCH"
+git checkout --quiet "$BASE_BRANCH"
+run_bench "$BASE_BRANCH"
+
+echo "==> Benchmarking $FEATURE_BRANCH"
+git checkout --quiet "$FEATURE_BRANCH"
+run_bench "$FEATURE_BRANCH"
+
+echo "==> Comparison"
+if command -v critcmp >/dev/null 2>&1; then
+    critcmp "$BASE_BRANCH" "$FEATURE_BRANCH"
+else
+    echo "critcmp not found; showing criterion's own comparison instead."
+    echo "Install it with: cargo install critcmp"
+    "${BENCH_CMD[@]}" --load-baseline "$FEATURE_BRANCH" --baseline "$BASE_BRANCH"
+fi