From dd725c5bb93fabcc4ff8e0a597d1cfbe037718a9 Mon Sep 17 00:00:00 2001 From: blaginin Date: Fri, 12 Sep 2025 10:17:03 +0100 Subject: [PATCH 1/5] Always run CI checks --- .asf.yaml | 24 +----------------------- .github/workflows/rust.yml | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 48e20755271e2..f5058c71e538e 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -53,31 +53,9 @@ github: required_status_checks: contexts: - "Check License Header" - - "cargo doc" - - "build and run with wasm-pack" - - "linux build test" - - "cargo test (macos-aarch64)" - - "Verify Vendored Code" - - "Check cargo fmt" - - "Verify MSRV (Min Supported Rust Version)" - - "cargo check datafusion-common features" - - "cargo check datafusion-substrait features" - - "cargo check datafusion-proto features" - - "cargo check datafusion features" - - "cargo check datafusion-functions features" - - "cargo test (amd64)" - - "cargo examples (amd64)" - - "cargo test doc (amd64)" - - "verify benchmark results (amd64)" - - "Run sqllogictest with Postgres runner" - - "Run sqllogictest in Substrait round-trip mode" - - "cargo test pyarrow (amd64)" - - "clippy" - - "check Cargo.toml formatting" - "Use prettier to check formatting of documents" - "check-files" - - "cargo test datafusion-cli (amd64)" - - "check configs.md and ***_functions.md is up-to-date" + - "Validate Rust Required Checks" # needs to be updated as part of the release process # .asf.yaml doesn't support wildcard branch protection rules, only exact branch names # https://github.com/apache/infrastructure-asfyaml?tab=readme-ov-file#branch-protection diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b064df2acb7c9..5431398894994 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -281,8 +281,8 @@ jobs: - name: Rust Dependency Cache uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 with: - save-if: ${{ github.ref_name == 'main' }} - shared-key: "amd-ci" + save-if: ${{ github.ref_name == 'main' }} + shared-key: "amd-ci" - name: Run tests (excluding doctests and datafusion-cli) env: RUST_BACKTRACE: 1 @@ -792,3 +792,36 @@ jobs: with: persist-credentials: false - uses: crate-ci/typos@v1 + + rust-required-checks: + name: Validate Rust Required Checks + needs: [ linux-rustdoc, + linux-wasm-pack, + linux-build-lib, + macos-aarch64, + vendor, + check-fmt, + msrv, + linux-datafusion-common-features, + linux-datafusion-substrait-features, + linux-datafusion-proto-features, + linux-cargo-check-datafusion, + linux-cargo-check-datafusion-functions, + linux-test, + linux-test-example, + linux-test-doc, + verify-benchmark-results, + sqllogictest-postgres, + sqllogictest-substrait, + test-datafusion-pyarrow, + clippy, + cargo-toml-formatting-checks, + linux-test-datafusion-cli, + config-docs-check + ] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: echo '${{ toJSON(needs) }}' | jq -e 'all(.[]; .result == "success" or .result == "skipped")' + From 6a0d050062fc75abcb478a18c1b47a2a0ae186d8 Mon Sep 17 00:00:00 2001 From: blaginin Date: Fri, 12 Sep 2025 10:31:17 +0100 Subject: [PATCH 2/5] Add `pre-job` --- .github/workflows/rust.yml | 85 +++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5431398894994..3e6970566c87f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,28 +23,30 @@ concurrency: on: push: - branches-ignore: - - 'gh-readonly-queue/**' - paths-ignore: - - "docs/**" - - "**.md" - - ".github/ISSUE_TEMPLATE/**" - - ".github/pull_request_template.md" pull_request: - paths-ignore: - - "docs/**" - - "**.md" - - ".github/ISSUE_TEMPLATE/**" - - ".github/pull_request_template.md" merge_group: # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: jobs: + pre-job: + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@v5 + with: + concurrent_skipping: 'never' + skip_after_successful_duplicate: 'false' + paths_ignore: '["**.md", "docs/**", ".github/ISSUE_TEMPLATE/**", ".github/pull_request_template.md"]' + # Check crate compiles and base cargo check passes linux-build-lib: name: linux build test + needs: pre-job + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -72,7 +74,8 @@ jobs: # subset of the features packages enabled. linux-datafusion-common-features: name: cargo check datafusion-common features - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -97,7 +100,8 @@ jobs: # subset of the features packages enabled. linux-datafusion-substrait-features: name: cargo check datafusion-substrait features - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -134,7 +138,8 @@ jobs: # subset of the features packages enabled. linux-datafusion-proto-features: name: cargo check datafusion-proto features - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -165,7 +170,8 @@ jobs: # subset of the features packages enabled. linux-cargo-check-datafusion: name: cargo check datafusion features - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -230,7 +236,8 @@ jobs: # subset of the features packages enabled. linux-cargo-check-datafusion-functions: name: cargo check datafusion-functions features - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -265,7 +272,8 @@ jobs: # Library and integration tests linux-test: name: cargo test (amd64) - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -304,7 +312,8 @@ jobs: # datafusion-cli tests linux-test-datafusion-cli: name: cargo test datafusion-cli (amd64) - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest # should be uncommented once https://github.com/apache/datafusion/pull/16644 is merged # and cache should be added @@ -332,7 +341,8 @@ jobs: linux-test-example: name: cargo examples (amd64) - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -362,7 +372,8 @@ jobs: # Run `cargo test doc` (test documentation examples) linux-test-doc: name: cargo test doc (amd64) - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -383,7 +394,8 @@ jobs: # Run `cargo doc` to ensure the rustdoc is clean linux-rustdoc: name: cargo doc - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -420,7 +432,8 @@ jobs: # verify that the benchmark queries return the correct results verify-benchmark-results: name: verify benchmark results (amd64) - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -453,7 +466,8 @@ jobs: sqllogictest-postgres: name: "Run sqllogictest with Postgres runner" - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -491,7 +505,8 @@ jobs: sqllogictest-substrait: name: "Run sqllogictest in Substrait round-trip mode" - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -562,7 +577,8 @@ jobs: test-datafusion-pyarrow: name: cargo test pyarrow (amd64) - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust:bullseye # Use the bullseye tag image which comes with python3.9 @@ -586,6 +602,8 @@ jobs: vendor: name: Verify Vendored Code + needs: pre-job + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -603,6 +621,8 @@ jobs: check-fmt: name: Check cargo fmt + needs: pre-job + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -661,7 +681,8 @@ jobs: clippy: name: clippy - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -686,7 +707,8 @@ jobs: cargo-toml-formatting-checks: name: check Cargo.toml formatting - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -707,7 +729,8 @@ jobs: config-docs-check: name: check configs.md and ***_functions.md is up-to-date - needs: linux-build-lib + needs: [linux-build-lib, pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -741,6 +764,8 @@ jobs: # - datafusion-cli msrv: name: Verify MSRV (Min Supported Rust Version) + needs: [pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@ -786,6 +811,8 @@ jobs: run: cargo msrv --output-format json --log-target stdout verify typos: name: Spell Check with Typos + needs: [pre-job] + if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 From 61b0db9c3498cbc1d311c2d3adb52c83f985da84 Mon Sep 17 00:00:00 2001 From: blaginin Date: Fri, 12 Sep 2025 10:32:55 +0100 Subject: [PATCH 3/5] For `config-docs-check`, we don't need pre-job --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3e6970566c87f..85b1f686c4df1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -729,7 +729,7 @@ jobs: config-docs-check: name: check configs.md and ***_functions.md is up-to-date - needs: [linux-build-lib, pre-job] + needs: [linux-build-lib] if: needs.pre-job.outputs.should_skip != 'true' runs-on: ubuntu-latest container: From 7b30fa444becd9b5d6db07bbd65614a8271283c6 Mon Sep 17 00:00:00 2001 From: blaginin Date: Fri, 12 Sep 2025 10:37:03 +0100 Subject: [PATCH 4/5] Comments --- .asf.yaml | 1 + .github/workflows/rust.yml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index f5058c71e538e..3b947c1622b99 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -55,6 +55,7 @@ github: - "Check License Header" - "Use prettier to check formatting of documents" - "check-files" + # This job marks steps in `.github/workflows/rust.yml` as required - "Validate Rust Required Checks" # needs to be updated as part of the release process # .asf.yaml doesn't support wildcard branch protection rules, only exact branch names diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 85b1f686c4df1..2c5aeabbb3e12 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -27,9 +27,14 @@ on: merge_group: # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow + # + # Note, we use merge queue for this repo. If you'd like to add a new "required" check, that should be checked before + # the PR is merged, you should add your step to `rust-required-checks` workflow_dispatch: jobs: + # This job checks if heavy-weight jobs can be skipped, because only doc or non-code files were changed + # More on this: https://github.com/orgs/community/discussions/45899 pre-job: runs-on: ubuntu-latest outputs: From cb327a7b202562ce69501f95b2dacc31eece33a4 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sun, 14 Sep 2025 13:28:21 +0100 Subject: [PATCH 5/5] keep `branches-ignore` --- .github/workflows/rust.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2c5aeabbb3e12..a440839fb2602 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,6 +23,8 @@ concurrency: on: push: + branches-ignore: + - 'gh-readonly-queue/**' pull_request: merge_group: # manual trigger