ci: Add merge queue trigger (#51833)

This adds a merge queue trigger to our main CI job to allow for a merge
queue which runs a subset of our tests before merging to main.

With this setup, merging to main would probably take around one to two
minutes and would not catch all but definitely most issues. We could
extend it here though as we'd like.

Release Notes:

- N/A

.github/workflows/run_tests.yml

@@ -6,6 +6,7 @@ env:
   RUST_BACKTRACE: '1'
   CARGO_INCREMENTAL: '0'
 on:
+  merge_group: {}
   pull_request:
     branches:
     - '**'
@@ -175,7 +176,7 @@ jobs:
   clippy_windows:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: self-32vcpu-windows-2022
     steps:
     - name: steps::checkout_repo
@@ -205,7 +206,7 @@ jobs:
   clippy_linux:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-16x32-ubuntu-2204
     env:
       CC: clang
@@ -243,7 +244,7 @@ jobs:
   clippy_mac:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-mac-large
     steps:
     - name: steps::checkout_repo
@@ -274,7 +275,7 @@ jobs:
   clippy_mac_x86_64:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-mac-large
     steps:
     - name: steps::checkout_repo
@@ -307,7 +308,7 @@ jobs:
   run_tests_windows:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: self-32vcpu-windows-2022
     steps:
     - name: steps::checkout_repo
@@ -349,7 +350,7 @@ jobs:
   run_tests_linux:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-16x32-ubuntu-2204
     env:
       CC: clang
@@ -407,7 +408,7 @@ jobs:
   run_tests_mac:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-mac-large
     steps:
     - name: steps::checkout_repo
@@ -450,7 +451,7 @@ jobs:
   doctests:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-16x32-ubuntu-2204
     env:
       CC: clang
@@ -494,7 +495,7 @@ jobs:
   check_workspace_binaries:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-8x16-ubuntu-2204
     env:
       CC: clang
@@ -538,7 +539,7 @@ jobs:
   build_visual_tests_binary:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-mac-large
     steps:
     - name: steps::checkout_repo
@@ -563,7 +564,7 @@ jobs:
   check_wasm:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-8x16-ubuntu-2204
     steps:
     - name: steps::checkout_repo
@@ -602,7 +603,7 @@ jobs:
   check_dependencies:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_tests == 'true'
+    if: needs.orchestrate.outputs.run_tests == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-2x4-ubuntu-2404
     env:
       CC: clang
@@ -634,7 +635,7 @@ jobs:
   check_docs:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_docs == 'true'
+    if: needs.orchestrate.outputs.run_docs == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-8x16-ubuntu-2204
     env:
       CC: clang
@@ -683,7 +684,7 @@ jobs:
   check_licenses:
     needs:
     - orchestrate
-    if: needs.orchestrate.outputs.run_licenses == 'true'
+    if: needs.orchestrate.outputs.run_licenses == 'true' && github.event_name != 'merge_group'
     runs-on: namespace-profile-2x4-ubuntu-2404
     steps:
     - name: steps::checkout_repo

tooling/xtask/src/tasks/workflows/extension_tests.rs

@@ -30,8 +30,8 @@ pub(crate) fn extension_tests() -> Workflow {
 
     let jobs = [
         orchestrate,
-        should_check_rust.guard(check_rust()),
-        should_check_extension.guard(check_extension()),
+        should_check_rust.and_always().then(check_rust()),
+        should_check_extension.and_always().then(check_extension()),
     ];
 
     let tests_pass = tests_pass(&jobs, &[]);

tooling/xtask/src/tasks/workflows/run_tests.rs

@@ -1,5 +1,5 @@
 use gh_workflow::{
-    Concurrency, Container, Event, Expression, Input, Job, Level, Permissions, Port, PullRequest,
+    Container, Event, Expression, Input, Job, Level, MergeGroup, Permissions, Port, PullRequest,
     Push, Run, Step, Strategy, Use, UsesJob, Workflow,
 };
 use indexmap::IndexMap;
@@ -48,26 +48,55 @@ pub(crate) fn run_tests() -> Workflow {
     let mut jobs = vec![
         orchestrate,
         check_style(),
-        should_run_tests.guard(clippy(Platform::Windows, None)),
-        should_run_tests.guard(clippy(Platform::Linux, None)),
-        should_run_tests.guard(clippy(Platform::Mac, None)),
-        should_run_tests.guard(clippy(Platform::Mac, Some(Arch::X86_64))),
-        should_run_tests.guard(run_platform_tests(Platform::Windows)),
-        should_run_tests.guard(run_platform_tests(Platform::Linux)),
-        should_run_tests.guard(run_platform_tests(Platform::Mac)),
-        should_run_tests.guard(doctests()),
-        should_run_tests.guard(check_workspace_binaries()),
-        should_run_tests.guard(build_visual_tests_binary()),
-        should_run_tests.guard(check_wasm()),
-        should_run_tests.guard(check_dependencies()), // could be more specific here?
-        should_check_docs.guard(check_docs()),
-        should_check_licences.guard(check_licenses()),
-        should_check_scripts.guard(check_scripts()),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(clippy(Platform::Windows, None)),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(clippy(Platform::Linux, None)),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(clippy(Platform::Mac, None)),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(clippy(Platform::Mac, Some(Arch::X86_64))),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(run_platform_tests(Platform::Windows)),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(run_platform_tests(Platform::Linux)),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(run_platform_tests(Platform::Mac)),
+        should_run_tests.and_not_in_merge_queue().then(doctests()),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(check_workspace_binaries()),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(build_visual_tests_binary()),
+        should_run_tests.and_not_in_merge_queue().then(check_wasm()),
+        should_run_tests
+            .and_not_in_merge_queue()
+            .then(check_dependencies()), // could be more specific here?
+        should_check_docs
+            .and_not_in_merge_queue()
+            .then(check_docs()),
+        should_check_licences
+            .and_not_in_merge_queue()
+            .then(check_licenses()),
+        should_check_scripts.and_always().then(check_scripts()),
     ];
     let ext_tests = extension_tests();
     let tests_pass = tests_pass(&jobs, &[&ext_tests.name]);
 
-    jobs.push(should_run_tests.guard(check_postgres_and_protobuf_migrations())); // could be more specific here?
+    // TODO: For merge queues, this should fail in the merge queue context
+    jobs.push(
+        should_run_tests
+            .and_always()
+            .then(check_postgres_and_protobuf_migrations()),
+    ); // could be more specific here?
 
     named::workflow()
         .add_event(
@@ -77,16 +106,10 @@ pub(crate) fn run_tests() -> Workflow {
                         .add_branch("main")
                         .add_branch("v[0-9]+.[0-9]+.x"),
                 )
-                .pull_request(PullRequest::default().add_branch("**")),
-        )
-        .concurrency(
-            Concurrency::default()
-                .group(concat!(
-                    "${{ github.workflow }}-${{ github.ref_name }}-",
-                    "${{ github.ref_name == 'main' && github.sha || 'anysha' }}"
-                ))
-                .cancel_in_progress(true),
+                .pull_request(PullRequest::default().add_branch("**"))
+                .merge_group(MergeGroup::default()),
         )
+        .concurrency(vars::one_workflow_per_non_main_branch())
         .add_env(("CARGO_TERM_COLOR", "always"))
         .add_env(("RUST_BACKTRACE", 1))
         .add_env(("CARGO_INCREMENTAL", 0))

tooling/xtask/src/tasks/workflows/vars.rs

@@ -1,4 +1,4 @@
-use std::cell::RefCell;
+use std::{cell::RefCell, ops::Not};
 
 use gh_workflow::{
     Concurrency, Env, Expression, Step, WorkflowCallInput, WorkflowCallSecret,
@@ -130,21 +130,50 @@ impl PathCondition {
             set_by_step: Default::default(),
         }
     }
-    pub fn guard(&self, job: NamedJob) -> NamedJob {
+
+    pub fn and_always<'a>(&'a self) -> PathContextCondition<'a> {
+        PathContextCondition {
+            condition: self,
+            run_in_merge_queue: true,
+        }
+    }
+
+    pub fn and_not_in_merge_queue<'a>(&'a self) -> PathContextCondition<'a> {
+        PathContextCondition {
+            condition: self,
+            run_in_merge_queue: false,
+        }
+    }
+}
+
+pub struct PathContextCondition<'a> {
+    condition: &'a PathCondition,
+    run_in_merge_queue: bool,
+}
+
+impl<'a> PathContextCondition<'a> {
+    pub fn then(&'a self, job: NamedJob) -> NamedJob {
         let set_by_step = self
+            .condition
             .set_by_step
             .borrow()
             .clone()
-            .unwrap_or_else(|| panic!("condition {},is never set", self.name));
+            .unwrap_or_else(|| panic!("condition {},is never set", self.condition.name));
         NamedJob {
             name: job.name,
-            job: job
-                .job
-                .add_need(set_by_step.clone())
-                .cond(Expression::new(format!(
-                    "needs.{}.outputs.{} == 'true'",
-                    &set_by_step, self.name
-                ))),
+            job: job.job.add_need(set_by_step.clone()).cond(Expression::new(
+                format!(
+                    "needs.{}.outputs.{} == 'true' {merge_queue_condition}",
+                    &set_by_step,
+                    self.condition.name,
+                    merge_queue_condition = self
+                        .run_in_merge_queue
+                        .not()
+                        .then_some("&& github.event_name != 'merge_group'")
+                        .unwrap_or_default()
+                )
+                .trim(),
+            )),
         }
     }
 }