Skip to content

Commit 6629a69

Browse files
borsbors
committed
Auto merge of #124366 - Kobzol:remove-yaml-expansion, r=<try>
CI: remove `expand-yaml-anchors` Opening for experiments. Blocked on #124332 (first 4 commits are from that PR). r? `@ghost`
2 parents 865808b + 0ef1a7e commit 6629a69

File tree

13 files changed

+137
-804
lines changed

13 files changed

+137
-804
lines changed

‎.github/workflows/ci.yml

+135-105
Original file line numberDiff line numberDiff line change
@@ -1,218 +1,248 @@
1-
#############################################################
2-
# WARNING: automatically generated file, DO NOT CHANGE! #
3-
#############################################################
4-
5-
# This file was automatically generated by the expand-yaml-anchors tool. The
6-
# source file that generated this one is:
7-
#
8-
# src/ci/github-actions/ci.yml
9-
#
10-
# Once you make changes to that file you need to run:
1+
# This file defines our primary CI workflow that runs on pull requests
2+
# and also on pushes to special branches (auto, try).
113
#
12-
# ./x.py run src/tools/expand-yaml-anchors/
13-
#
14-
# The CI build will fail if the tool is not run after changes to this file.
4+
# The actual definition of the executed jobs is calculated by a Python
5+
# script located at src/ci/github-actions/calculate-job-matrix.py, which
6+
# uses job definition data from src/ci/github-actions/jobs.yml.
7+
# You should primarily modify the `jobs.yml` file if you want to modify
8+
# what jobs are executed in CI.
159

16-
---
1710
name: CI
18-
"on":
11+
on:
1912
push:
2013
branches:
2114
- auto
2215
- try
2316
- try-perf
2417
- automation/bors/try
25-
- master
2618
pull_request:
2719
branches:
2820
- "**"
21+
2922
permissions:
3023
contents: read
3124
packages: write
25+
3226
defaults:
3327
run:
28+
# On Linux, macOS, and Windows, use the system-provided bash as the default
29+
# shell. (This should only make a difference on Windows, where the default
30+
# shell is PowerShell.)
3431
shell: bash
32+
3533
concurrency:
36-
group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}"
34+
# For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
35+
# We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
36+
# are all triggered on the same branch, but which should be able to run concurrently.
37+
group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
3738
cancel-in-progress: true
39+
3840
jobs:
41+
# The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
42+
# It calculates which jobs should be executed, based on the data of the ${{ github }} context.
43+
# If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
3944
calculate_matrix:
4045
name: Calculate job matrix
4146
runs-on: ubuntu-latest
4247
outputs:
43-
jobs: "${{ steps.jobs.outputs.jobs }}"
48+
jobs: ${{ steps.jobs.outputs.jobs }}
4449
steps:
4550
- name: Checkout the source code
4651
uses: actions/checkout@v4
4752
- name: Calculate the CI job matrix
4853
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
4954
id: jobs
5055
job:
51-
name: "${{ matrix.name }}"
52-
needs:
53-
- calculate_matrix
56+
name: ${{ matrix.name }}
57+
needs: [ calculate_matrix ]
58+
runs-on: "${{ matrix.os }}"
59+
defaults:
60+
run:
61+
shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
62+
timeout-minutes: 600
5463
env:
55-
CI_JOB_NAME: "${{ matrix.image }}"
64+
CI_JOB_NAME: ${{ matrix.image }}
5665
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
57-
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
58-
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
66+
# commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
67+
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
68+
DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5969
SCCACHE_BUCKET: rust-lang-ci-sccache2
60-
TOOLSTATE_REPO: "https://github.com./rust-lang-nursery/rust-toolstate"
70+
TOOLSTATE_REPO: https://github.com./rust-lang-nursery/rust-toolstate
6171
CACHE_DOMAIN: ci-caches.rust-lang.org
62-
continue-on-error: "${{ matrix.continue_on_error || false }}"
72+
continue-on-error: ${{ matrix.continue_on_error || false }}
6373
strategy:
6474
matrix:
65-
include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
66-
if: "fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null"
67-
defaults:
68-
run:
69-
shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
70-
timeout-minutes: 600
71-
runs-on: "${{ matrix.os }}"
75+
# Check the `calculate_matrix` job to see how is the matrix defined.
76+
include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
77+
# GitHub Actions fails the workflow if an empty list of jobs is provided to
78+
# the workflow, so we need to skip this job if nothing was produced by
79+
# the Python script.
80+
#
81+
# Unfortunately checking whether a list is empty is not possible in a nice
82+
# way due to GitHub Actions expressions limits.
83+
# This hack is taken from https://github.com./ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
84+
if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null
7285
steps:
73-
- if: "contains(matrix.os, 'windows')"
86+
- if: contains(matrix.os, 'windows')
7487
uses: msys2/[email protected]
7588
with:
76-
msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
89+
# i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
90+
msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
91+
# don't try to download updates for already installed packages
7792
update: false
93+
# don't try to use the msys that comes built-in to the github runner,
94+
# so we can control what is installed (i.e. not python)
7895
release: true
96+
# Inherit the full path from the Windows environment, with MSYS2's */bin/
97+
# dirs placed in front. This lets us run Windows-native Python etc.
7998
path-type: inherit
80-
install: "make dos2unix diffutils\n"
99+
install: >
100+
make
101+
dos2unix
102+
diffutils
103+
81104
- name: disable git crlf conversion
82105
run: git config --global core.autocrlf false
106+
83107
- name: checkout the source code
84108
uses: actions/checkout@v4
85109
with:
86110
fetch-depth: 2
111+
112+
# Rust Log Analyzer can't currently detect the PR number of a GitHub
113+
# Actions build on its own, so a hint in the log message is needed to
114+
# point it in the right direction.
87115
- name: configure the PR in which the error message will be posted
88-
run: "echo \"[CI_PR_NUMBER=$num]\""
116+
run: echo "[CI_PR_NUMBER=$num]"
89117
env:
90-
num: "${{ github.event.number }}"
91-
if: "success() && github.event_name == 'pull_request'"
118+
num: ${{ github.event.number }}
119+
if: success() && github.event_name == 'pull_request'
120+
92121
- name: add extra environment variables
93122
run: src/ci/scripts/setup-environment.sh
94123
env:
95-
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
124+
# Since it's not possible to merge `${{ matrix.env }}` with the other
125+
# variables in `job.<name>.env`, the variables defined in the matrix
126+
# are passed to the `setup-environment.sh` script encoded in JSON,
127+
# which then uses log commands to actually set them.
128+
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
129+
96130
- name: ensure the channel matches the target branch
97131
run: src/ci/scripts/verify-channel.sh
132+
98133
- name: collect CPU statistics
99134
run: src/ci/scripts/collect-cpu-stats.sh
135+
100136
- name: show the current environment
101137
run: src/ci/scripts/dump-environment.sh
138+
102139
- name: install awscli
103140
run: src/ci/scripts/install-awscli.sh
141+
104142
- name: install sccache
105143
run: src/ci/scripts/install-sccache.sh
144+
106145
- name: select Xcode
107146
run: src/ci/scripts/select-xcode.sh
147+
108148
- name: install clang
109149
run: src/ci/scripts/install-clang.sh
150+
110151
- name: install tidy
111152
run: src/ci/scripts/install-tidy.sh
153+
112154
- name: install WIX
113155
run: src/ci/scripts/install-wix.sh
156+
114157
- name: disable git crlf conversion
115158
run: src/ci/scripts/disable-git-crlf-conversion.sh
159+
116160
- name: checkout submodules
117161
run: src/ci/scripts/checkout-submodules.sh
162+
118163
- name: install MSYS2
119164
run: src/ci/scripts/install-msys2.sh
165+
120166
- name: install MinGW
121167
run: src/ci/scripts/install-mingw.sh
168+
122169
- name: install ninja
123170
run: src/ci/scripts/install-ninja.sh
171+
124172
- name: enable ipv6 on Docker
125173
run: src/ci/scripts/enable-docker-ipv6.sh
174+
175+
# Disable automatic line ending conversion (again). On Windows, when we're
176+
# installing dependencies, something switches the git configuration directory or
177+
# re-enables autocrlf. We've not tracked down the exact cause -- and there may
178+
# be multiple -- but this should ensure submodules are checked out with the
179+
# appropriate line endings.
126180
- name: disable git crlf conversion
127181
run: src/ci/scripts/disable-git-crlf-conversion.sh
182+
128183
- name: ensure line endings are correct
129184
run: src/ci/scripts/verify-line-endings.sh
185+
130186
- name: ensure backported commits are in upstream branches
131187
run: src/ci/scripts/verify-backported-commits.sh
188+
132189
- name: ensure the stable version number is correct
133190
run: src/ci/scripts/verify-stable-version-number.sh
191+
134192
- name: run the build
193+
# Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
135194
run: src/ci/scripts/run-build-from-ci.sh 2>&1
136195
env:
137-
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
138-
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
139-
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
196+
AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
197+
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
198+
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
199+
140200
- name: create github artifacts
141201
run: src/ci/scripts/create-doc-artifacts.sh
202+
142203
- name: upload artifacts to github
143204
uses: actions/upload-artifact@v4
144205
with:
145-
name: "${{ env.DOC_ARTIFACT_NAME }}"
206+
# name is set in previous step
207+
name: ${{ env.DOC_ARTIFACT_NAME }}
146208
path: obj/artifacts/doc
147209
if-no-files-found: ignore
148210
retention-days: 5
211+
149212
- name: upload artifacts to S3
150213
run: src/ci/scripts/upload-artifacts.sh
151214
env:
152-
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
153-
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
154-
if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
155-
master:
156-
name: master
215+
AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
216+
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
217+
# Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
218+
# builders *should* have the AWS credentials available. Still, explicitly
219+
# adding the condition is helpful as this way CI will not silently skip
220+
# deploying artifacts from a dist builder if the variables are misconfigured,
221+
# erroring about invalid credentials instead.
222+
if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
223+
224+
# This job isused to tell bors the final status of the build, as there is no practical way to detect
225+
# when a workflow is successful listening to webhooks only in our current bors implementation (homu).
226+
outcome:
227+
name: bors build finished
157228
runs-on: ubuntu-latest
158-
env:
159-
SCCACHE_BUCKET: rust-lang-ci-sccache2
160-
DEPLOY_BUCKET: rust-lang-ci2
161-
TOOLSTATE_REPO: "https://github.com./rust-lang-nursery/rust-toolstate"
162-
TOOLSTATE_ISSUES_API_URL: "https://api.github.com./repos/rust-lang/rust/issues"
163-
TOOLSTATE_PUBLISH: 1
164-
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
165-
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
166-
AWS_REGION: us-west-1
167-
CACHE_DOMAIN: ci-caches.rust-lang.org
168-
if: "github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'"
229+
needs: [ job ]
230+
# !cancelled() executes the job regardless of whether the previous jobs passed or failed
231+
if: "!cancelled() && github.event_name == 'push'"
169232
steps:
170-
- name: checkout the source code
171-
uses: actions/checkout@v4
172-
with:
173-
fetch-depth: 2
233+
# Calculate the exit status of the whole CI workflow (0 if all dependent jobs were either successful
234+
# or skipped, otherwise 1).
235+
- name: calculate the correct exit status
236+
id: status
237+
run: |
238+
jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
239+
echo "status=$?" >> $GITHUB_OUTPUT
240+
# Publish the toolstate if an auto build succeeds (just before push to master)
174241
- name: publish toolstate
175242
run: src/ci/publish_toolstate.sh
176243
shell: bash
244+
if: steps.outputs.status == 0 && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
177245
env:
178-
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
179-
try-success:
180-
needs:
181-
- job
182-
if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
183-
steps:
184-
- name: mark the job as a success
185-
run: exit 0
186-
shell: bash
187-
name: bors build finished
188-
runs-on: ubuntu-latest
189-
try-failure:
190-
needs:
191-
- job
192-
if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
193-
steps:
194-
- name: mark the job as a failure
195-
run: exit 1
196-
shell: bash
197-
name: bors build finished
198-
runs-on: ubuntu-latest
199-
auto-success:
200-
needs:
201-
- job
202-
if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
203-
steps:
204-
- name: mark the job as a success
205-
run: exit 0
206-
shell: bash
207-
name: bors build finished
208-
runs-on: ubuntu-latest
209-
auto-failure:
210-
needs:
211-
- job
212-
if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
213-
steps:
214-
- name: mark the job as a failure
215-
run: exit 1
216-
shell: bash
217-
name: bors build finished
218-
runs-on: ubuntu-latest
246+
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
247+
- name: set the correct exit status
248+
run: exit ${{ steps.outputs.status == 0 }}

0 commit comments

Comments
 (0)