feat: update the benchmark dataset and generator to add runner to the data id

.github/workflows/sql-benchmarks.yml

@@ -377,6 +377,7 @@ jobs:
             --targets-json '${{ steps.targets.outputs.targets_json }}' \
             --output results.json \
             --no-build \
+            --runner "ec2_${{ inputs.machine_type }}" \
             ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \
             ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }}
 
@@ -395,6 +396,7 @@ jobs:
             --targets-json '${{ steps.targets.outputs.targets_json }}' \
             --output results.json \
             --no-build \
+            --runner "ec2_${{ inputs.machine_type }}" \
             ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \
             --opt remote-data-dir=${{ matrix.remote_storage }} \
             ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }}

bench-orchestrator/bench_orchestrator/cli.py

@@ -202,6 +202,10 @@ def run(
         str | None,
         typer.Option("--targets-json", help="Exact benchmark targets as a JSON array"),
     ] = None,
+    runner: Annotated[
+        str | None,
+        typer.Option("--runner", help="Benchmark runner ID (e.g., ec2_c6id.8xlarge)"),
+    ] = None,
     output: Annotated[
         Path | None,
         typer.Option("--output", help="Optional path for compatibility JSONL output"),
@@ -289,6 +293,7 @@ def run(
                         samply=samply,
                         sample_rate=sample_rate,
                         tracing=tracing,
+                        runner=runner,
                         on_result=lambda line, store_writer=ctx.write_raw_json, compatibility=compatibility_file: (
                             write_result_line(
                                 line,

bench-orchestrator/bench_orchestrator/comparison/analyzer.py

@@ -10,6 +10,9 @@
 import numpy as np
 import pandas as pd
 
+OLD_QUERY_NAME = re.compile(r"_q(\d+)/")
+NEW_QUERY_NAME = re.compile(r"/q(\d+)(?::memory|/memory)?/")
+
 
 @dataclass
 class TargetRef:
@@ -64,9 +67,14 @@ def extract_target_fields(df: pd.DataFrame) -> pd.DataFrame:
 
     # Extract query number from name if present
     if "name" in df.columns:
-        # Pattern: dataset_qNN/engine:format
-        pattern = r"_q(\d+)/"
-        df["query"] = df["name"].apply(lambda n: int(m.group(1)) if (m := re.search(pattern, str(n))) else None)
+        # Patterns:
+        # - dataset_qNN/engine:format
+        # - dataset/.../qNN/engine:format
+        df["query"] = df["name"].apply(
+            lambda n: int(m.group(1))
+            if (m := (OLD_QUERY_NAME.search(str(n)) or NEW_QUERY_NAME.search(str(n))))
+            else None
+        )
 
     return df
 

bench-orchestrator/bench_orchestrator/runner/executor.py

@@ -39,6 +39,7 @@ def build_command(
         samply: bool = False,
         sample_rate: int | None = None,
         tracing: bool = False,
+        runner: str | None = None,
     ) -> list[str]:
         """Build the command used to execute a benchmark binary."""
         cmd = [
@@ -64,6 +65,8 @@ def build_command(
             cmd.append("--track-memory")
         if tracing:
             cmd.append("--tracing")
+        if runner:
+            cmd.extend(["--runner", runner])
         if options:
             for key, value in options.items():
                 cmd.extend(["--opt", f"{key}={value}"])
@@ -94,6 +97,7 @@ def run(
         samply: bool = False,
         sample_rate: int | None = None,
         tracing: bool = False,
+        runner: str | None = None,
         on_result: Callable[[str], None] | None = None,
     ) -> list[str]:
         """
@@ -123,6 +127,7 @@ def run(
             samply=samply,
             sample_rate=sample_rate,
             tracing=tracing,
+            runner=runner,
         )
 
         if self.verbose:

benchmarks-website/server.js

@@ -22,6 +22,8 @@ const COMMITS_URL =
 const REFRESH_INTERVAL = process.env.REFRESH_INTERVAL || 5 * 60 * 1000;
 const MAX_POINTS = 200;
 const USE_LOCAL_DATA = process.env.USE_LOCAL_DATA === "true";
+const LOCAL_DATA_FILE = process.env.LOCAL_DATA_FILE || null;
+const LOCAL_COMMITS_FILE = process.env.LOCAL_COMMITS_FILE || null;
 
 // Benchmark groups: non-query groups + simple suites + fan-out suites
 const GROUPS = [
@@ -64,8 +66,50 @@ const geoMean = (arr) =>
       )
     : null;
 
+function queryLabel(suite, queryNumber) {
+  return `${suite.queryPrefix} Q${queryNumber}`;
+}
+
+function parseSqlBenchmarkId(id) {
+  let parts = id.split("/");
+  if (parts[0]?.toLowerCase() === "memory") parts = parts.slice(1);
+  if (parts.length < 4) return null;
+
+  const suitePrefix = parts[0].toLowerCase();
+  const suite = QUERY_SUITES.find(
+    (querySuite) => querySuite.prefix.toLowerCase() === suitePrefix,
+  );
+  if (!suite) return null;
+
+  const querySegment = parts.at(-3);
+  const runner = parts.at(-2);
+  const seriesName = parts.at(-1);
+  const queryMatch = querySegment?.match(/^q(\d+)$/i);
+  if (!queryMatch || !runner || !seriesName?.includes(":")) return null;
+
+  const queryNumber = parseInt(queryMatch[1], 10);
+  return {
+    suite,
+    queryNumber,
+    datasetSegments: parts.slice(1, -3),
+    chartName: queryLabel(suite, queryNumber),
+    seriesName,
+    sortPosition: queryNumber,
+  };
+}
+
+function scaleFactorFromBenchmarkId(parsedSqlId) {
+  const sfSegment = parsedSqlId.datasetSegments.find((segment) =>
+    segment.toLowerCase().startsWith("sf_"),
+  );
+  if (!sfSegment) return 1;
+
+  const sf = parseFloat(sfSegment.slice(3).replace(/_/g, "."));
+  return Number.isFinite(sf) ? Math.round(sf) : 1;
+}
+
 // Categorize benchmarks based on name patterns and metadata
-function getGroup(benchmark) {
+function getGroup(benchmark, parsedSqlId = parseSqlBenchmarkId(benchmark.name)) {
   const name = benchmark.name;
   const lower = name.toLowerCase();
 
@@ -105,26 +149,20 @@ function getGroup(benchmark) {
     return "Compression";
   }
 
-  // SQL query suites: match "{prefix}_q..." or "{prefix}/..."
-  for (const suite of QUERY_SUITES) {
-    if (
-      !lower.startsWith(suite.prefix + "_q") &&
-      !lower.startsWith(suite.prefix + "/")
-    )
-      continue;
+  if (parsedSqlId) {
+    const suite = parsedSqlId.suite;
     if (suite.skip) return null;
     if (!suite.fanOut) return suite.displayName;
     // Fan-out suites: expand by storage and scale factor
     const storage = benchmark.storage?.toUpperCase() === "S3" ? "S3" : "NVMe";
-    const rawSf = benchmark.dataset?.[suite.datasetKey]?.scale_factor;
-    const sf = rawSf ? Math.round(parseFloat(rawSf)) : 1;
+    const sf = scaleFactorFromBenchmarkId(parsedSqlId);
     return `${suite.displayName} (${storage}) (SF=${sf})`;
   }
 
   return null;
 }
 
-// Format query name for display: "{prefix}_q00" -> "{QUERY_PREFIX} Q0"
+// Format query name for display: "{prefix}_q00" -> "{QUERY_PREFIX} Q0".
 function formatQuery(q) {
   const lower = q.toLowerCase();
   for (const suite of QUERY_SUITES) {
@@ -232,7 +270,14 @@ function readLocalJsonl(fp) {
 async function forEachBenchmark(callback) {
   let stream;
   if (USE_LOCAL_DATA) {
-    stream = fs.createReadStream(path.join(__dirname, "sample/data.json"));
+    const localDataFile =
+      LOCAL_DATA_FILE ||
+      (fs.existsSync(path.join(__dirname, "sample/data.json"))
+        ? path.join(__dirname, "sample/data.json")
+        : path.join(__dirname, "sample/data.json.gz"));
+    stream = localDataFile.endsWith(".gz")
+      ? fs.createReadStream(localDataFile).pipe(zlib.createGunzip())
+      : fs.createReadStream(localDataFile);
   } else {
     const res = await fetch(DATA_URL);
     if (!res.ok) throw new Error(`Fetch failed: ${DATA_URL} ${res.status}`);
@@ -259,7 +304,9 @@ async function refresh() {
   try {
     // Load commits first (small dataset, must be fully in memory for indexing)
     const commitsArr = USE_LOCAL_DATA
-      ? await readLocalJsonl(path.join(__dirname, "sample/commits.json"))
+      ? await readLocalJsonl(
+          LOCAL_COMMITS_FILE || path.join(__dirname, "sample/commits.json"),
+        )
       : await fetchJsonl(COMMITS_URL);
 
     // Build commit index (O(1) lookup)
@@ -283,27 +330,39 @@ async function refresh() {
         return;
       }
 
-      const group = getGroup(b);
+      const parsedSqlId = parseSqlBenchmarkId(b.name);
+      const group = getGroup(b, parsedSqlId);
       if (!group) {
-        uncategorized.add(b.name.split("/")[0]);
+        if (!parsedSqlId?.suite.skip) {
+          uncategorized.add(b.name.split("/")[0]);
+        }
         return;
       }
       if (!groups[group]) return;
 
       // Random access names have the form: random-access/{dataset}/{pattern}/{format}
       // Historical random access names: random-access/{format}
       // Other benchmarks use: {query}/{series}
-      let seriesName, chartName;
+      let seriesName, chartName, sortPos;
       const parts = b.name.split("/");
       if (group === "Random Access" && parts.length === 4) {
         chartName = `${parts[1]}/${parts[2]}`.toUpperCase().replace(/[_-]/g, " ");
         seriesName = rename(parts[3] || "default");
+        sortPos = 0;
       } else if (group === "Random Access" && parts.length === 2) {
         chartName = "RANDOM ACCESS";
         seriesName = rename(parts[1] || "default");
+        sortPos = 0;
+      } else if (parsedSqlId) {
+        seriesName = rename(parsedSqlId.seriesName);
+        chartName = parsedSqlId.chartName;
+        sortPos = parsedSqlId.sortPosition;
       } else {
         seriesName = rename(parts[1] || "default");
         chartName = formatQuery(parts[0]);
+        sortPos = parts[0].match(/q(\d+)$/i)?.[1]
+          ? parseInt(RegExp.$1, 10)
+          : 0;
       }
       chartName = normalizeChartName(group, chartName);
       if (chartName.includes("PARQUET-UNC")) return;
@@ -318,9 +377,6 @@ async function refresh() {
         else unit = "ns";
       }
 
-      const sortPos = parts[0].match(/q(\d+)$/i)?.[1]
-        ? parseInt(RegExp.$1, 10)
-        : 0;
       const idx = commitIdx.get(commit.id);
       if (idx === undefined) return;
 

benchmarks-website/src/config.js

@@ -46,6 +46,7 @@ export const QUERY_SUITES = [
     fanOut: true,
   },
   { prefix: "fineweb", skip: true },
+  { prefix: "gharchive", skip: true },
 ];
 
 // Pre-registered fan-out groups (storage x scale factor).

benchmarks/datafusion-bench/src/main.rs

@@ -94,6 +94,9 @@ struct Args {
     #[arg(long, default_value_t = false)]
     track_memory: bool,
 
+    #[arg(long, default_value = "unknown")]
+    runner: String,
+
     #[arg(long, default_value_t = false)]
     explain: bool,
 
@@ -149,6 +152,7 @@ async fn main() -> anyhow::Result<()> {
     let mut runner = SqlBenchmarkRunner::new(
         &*benchmark,
         Engine::DataFusion,
+        args.runner.clone(),
         args.formats.clone(),
         args.track_memory,
         args.hide_progress_bar,

benchmarks/duckdb-bench/src/main.rs

@@ -64,6 +64,9 @@ struct Args {
     #[arg(long, default_value_t = false)]
     hide_progress_bar: bool,
 
+    #[arg(long, default_value = "unknown")]
+    runner: String,
+
     #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))]
     formats: Vec<Format>,
 
@@ -142,6 +145,7 @@ fn main() -> anyhow::Result<()> {
     let mut runner = SqlBenchmarkRunner::new(
         &*benchmark,
         Engine::DuckDB,
+        args.runner.clone(),
         args.formats.clone(),
         args.track_memory,
         args.hide_progress_bar,

benchmarks/lance-bench/src/main.rs

@@ -65,6 +65,9 @@ struct Args {
     #[arg(long, default_value_t = false)]
     track_memory: bool,
 
+    #[arg(long, default_value = "unknown")]
+    runner: String,
+
     #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))]
     options: Vec<Opt>,
 }
@@ -93,6 +96,7 @@ async fn main() -> anyhow::Result<()> {
     let mut runner = SqlBenchmarkRunner::new(
         &*benchmark,
         Engine::DataFusion,
+        args.runner.clone(),
         vec![Format::Lance],
         args.track_memory,
         args.hide_progress_bar,