Skip to content

Commit 78aa022

Browse files
borsbors
committed
Auto merge of #136247 - marcoieni:free-more-space-ubuntu-24, r=<try>
[experiment] ci: free more space in ubuntu 24 free runners try-job: x86_64-gnu-debug try-job: armhf-gnu try-job: dist-powerpc64le-linux try-job: dist-ohos try-job: aarch64-gnu
2 parents 5a45ab9 + 1b360c8 commit 78aa022

File tree

2 files changed

+194
-92
lines changed

2 files changed

+194
-92
lines changed

‎src/ci/github-actions/jobs.yml

+4-9
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,11 @@ runners:
55
env: { }
66

77
- &job-linux-4c
8-
os: ubuntu-22.04
8+
os: ubuntu-24.04
99
# Free some disk space to avoid running out of space during the build.
1010
free_disk: true
1111
<<: *base-job
1212

13-
# Large runner used mainly for its bigger disk capacity
14-
- &job-linux-4c-largedisk
15-
os: ubuntu-22.04-4core-16gb
16-
<<: *base-job
17-
1813
- &job-linux-8c
1914
os: ubuntu-22.04-8core-32gb
2015
<<: *base-job
@@ -46,7 +41,7 @@ runners:
4641
- &job-aarch64-linux
4742
# Free some disk space to avoid running out of space during the build.
4843
free_disk: true
49-
os: ubuntu-22.04-arm
44+
os: ubuntu-24.04-arm
5045

5146
- &job-aarch64-linux-8c
5247
os: ubuntu-22.04-arm64-8core-32gb
@@ -178,7 +173,7 @@ auto:
178173
<<: *job-linux-4c
179174

180175
- name: dist-powerpc64le-linux
181-
<<: *job-linux-4c-largedisk
176+
<<: *job-linux-4c
182177

183178
- name: dist-riscv64-linux
184179
<<: *job-linux-4c
@@ -291,7 +286,7 @@ auto:
291286
- name: x86_64-gnu-debug
292287
# This seems to be needed because a full stage 2 build + run-make tests
293288
# overwhelms the storage capacity of the standard 4c runner.
294-
<<: *job-linux-4c-largedisk
289+
<<: *job-linux-4c
295290

296291
- name: x86_64-gnu-distcheck
297292
<<: *job-linux-8c

‎src/ci/scripts/free-disk-space.sh

+190-83
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,74 @@
11
#!/bin/bash
2+
set -euo pipefail
23

3-
# Free disk space on Linux GitHub action runners
4+
# Free disk space on Linux GitHub action runners.
45
# Script inspired by https://github.com./jlumbroso/free-disk-space
56

6-
# print a line of the specified character
7+
# When updating to a new ubuntu version (e.g. from ubuntu-24.04):
8+
# - Check that there are no docker images preinstalled with `docker image ls`
9+
# - Check that there are no big packages preinstalled that we aren't using
10+
# - Check that all directores we are removing are still present (look at the warnings)
11+
12+
# Print a line of the specified character.
713
printSeparationLine() {
814
for ((i = 0; i < 80; i++)); do
915
printf "%s" "$1"
1016
done
1117
printf "\n"
1218
}
1319

14-
# compute available space
20+
# Compute available space.
1521
# REF: https://unix.stackexchange.com/a/42049/60849
1622
# REF: https://stackoverflow.com/a/450821/408734
17-
getAvailableSpace() { echo $(df -a | awk 'NR > 1 {avail+=$4} END {print avail}'); }
23+
getAvailableSpace() {
24+
df -a | awk 'NR > 1 {avail+=$4} END {print avail}'
25+
}
1826

19-
# make Kb human readable (assume the input is Kb)
27+
# Make Kb human readable (assume the input is Kb).
2028
# REF: https://unix.stackexchange.com/a/44087/60849
21-
formatByteCount() { echo $(numfmt --to=iec-i --suffix=B --padding=7 $1'000'); }
29+
formatByteCount() {
30+
numfmt --to=iec-i --suffix=B --padding=7 "$1"'000'
31+
}
32+
33+
# Check if the architecture is x86.
34+
isX86() {
35+
local arch
36+
arch=$(uname -m)
37+
if [ "$arch" = "x86_64" ]; then
38+
return 0
39+
else
40+
return 1
41+
fi
42+
}
43+
44+
# Execute a task, printing how much space was saved and how long it took to run the task
45+
execAndMeasure() {
46+
local task_name=${1}
2247

23-
# macro to output saved space
24-
printSavedSpace() {
25-
# Disk space before the operation
26-
local before=${1}
27-
local title=${2:-}
48+
local start
49+
start=$(date +%s)
50+
51+
local before
52+
before=$(getAvailableSpace)
53+
54+
# Run the task. Skip the first argument because it's the task name.
55+
"${@:2}"
56+
57+
local end
58+
end=$(date +%s)
2859

2960
local after
3061
after=$(getAvailableSpace)
62+
63+
# How much space was saved.
3164
local saved=$((after - before))
65+
# How long the task took.
66+
local seconds_taken=$((end - start))
3267

33-
echo ""
34-
printSeparationLine "*"
35-
if [ -n "${title}" ]; then
36-
echo "=> ${title}: Saved $(formatByteCount "$saved")"
37-
else
38-
echo "=> Saved $(formatByteCount "$saved")"
39-
fi
40-
printSeparationLine "*"
41-
echo ""
68+
echo "==> ${task_name}: Saved $(formatByteCount "$saved") in $seconds_taken seconds"
4269
}
4370

44-
# macro to print output of df with caption
71+
# Print output of df with caption. It shows information about disk space.
4572
printDF() {
4673
local caption=${1}
4774

@@ -52,91 +79,171 @@ printDF() {
5279
echo ""
5380
df -h
5481
printSeparationLine "="
82+
echo ""
5583
}
5684

57-
removeDir() {
58-
dir=${1}
59-
60-
local before
61-
before=$(getAvailableSpace)
62-
63-
sudo rm -rf "$dir" || true
64-
65-
printSavedSpace "$before" "$dir"
85+
removeUnusedDirsAndFiles() {
86+
local to_remove=(
87+
"/etc/mysql"
88+
"/usr/local/aws-sam-cli"
89+
"/usr/local/doc/cmake"
90+
"/usr/local/julia"*
91+
"/usr/local/lib/android"
92+
"/usr/local/share/chromedriver-"*
93+
"/usr/local/share/chromium"
94+
"/usr/local/share/cmake-"*
95+
"/usr/local/share/edge_driver"
96+
"/usr/local/share/gecko_driver"
97+
"/usr/local/share/icons"
98+
"/usr/local/share/vim"
99+
"/usr/local/share/emacs"
100+
"/usr/local/share/powershell"
101+
"/usr/local/share/vcpkg"
102+
"/usr/share/apache-maven-"*
103+
"/usr/share/gradle-"*
104+
"/usr/share/java"
105+
"/usr/share/kotlinc"
106+
"/usr/share/miniconda"
107+
"/usr/share/php"
108+
"/usr/share/ri"
109+
"/usr/share/swift"
110+
111+
# binaries
112+
"/usr/local/bin/azcopy"
113+
"/usr/local/bin/bicep"
114+
"/usr/local/bin/ccmake"
115+
"/usr/local/bin/cmake-"*
116+
"/usr/local/bin/cmake"
117+
"/usr/local/bin/cpack"
118+
"/usr/local/bin/ctest"
119+
"/usr/local/bin/helm"
120+
"/usr/local/bin/kind"
121+
"/usr/local/bin/kustomize"
122+
"/usr/local/bin/minikube"
123+
"/usr/local/bin/packer"
124+
"/usr/local/bin/phpunit"
125+
"/usr/local/bin/pulumi-"*
126+
"/usr/local/bin/pulumi"
127+
"/usr/local/bin/stack"
128+
129+
# Haskell runtime
130+
"/usr/local/.ghcup"
131+
132+
# Azure
133+
"/opt/az"
134+
"/usr/share/az_"*
135+
136+
# Environemnt variable set by GitHub Actions
137+
"$AGENT_TOOLSDIRECTORY"
138+
)
139+
140+
for element in "${to_remove[@]}"; do
141+
if [ ! -e "$element" ]; then
142+
echo "::warning::Directory or file $element does not exist, skipping."
143+
else
144+
execAndMeasure "Removed $element" sudo rm -rf "$element"
145+
fi
146+
done
66147
}
67148

68-
execAndMeasureSpaceChange() {
69-
local operation=${1} # Function to execute
70-
local title=${2}
71-
72-
local before
73-
before=$(getAvailableSpace)
74-
$operation
75-
76-
printSavedSpace "$before" "$title"
149+
removeNodeModules() {
150+
sudo npm uninstall -g \
151+
"@bazel/bazelisk" \
152+
"bazel" \
153+
"grunt" \
154+
"gulp" \
155+
"lerna" \
156+
"n" \
157+
"newman" \
158+
"parcel" \
159+
"typescript" \
160+
"webpack-cli" \
161+
"webpack" \
162+
"yarn"
77163
}
78164

79-
# Remove large packages
80-
# REF: https://github.com./apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
165+
# Remove unused packages.
81166
cleanPackages() {
82-
sudo apt-get -qq remove -y --fix-missing \
83-
'^aspnetcore-.*' \
84-
'^dotnet-.*' \
85-
'^llvm-.*' \
86-
'php.*' \
87-
'^mongodb-.*' \
88-
'^mysql-.*' \
89-
'azure-cli' \
90-
'google-chrome-stable' \
91-
'firefox' \
92-
'powershell' \
93-
'mono-devel' \
94-
'libgl1-mesa-dri' \
95-
'google-cloud-sdk' \
96-
'google-cloud-cli'
167+
local packages=(
168+
'.*-icon-theme$'
169+
'^aspnetcore-.*'
170+
'^dotnet-.*'
171+
'^java-*'
172+
'^libllvm.*'
173+
'^llvm-.*'
174+
'^mercurial.*'
175+
'^mysql-.*'
176+
'^vim.*'
177+
'^fonts-.*'
178+
'azure-cli'
179+
'buildah'
180+
'cpp-13'
181+
'firefox'
182+
'gcc-12'
183+
'gcc-13'
184+
'gcc-14'
185+
'gcc'
186+
'g++-14'
187+
'gfortran-14'
188+
'groff-base'
189+
'kubectl'
190+
'libgl1-mesa-dri'
191+
'microsoft-edge-stable'
192+
'php.*'
193+
'podman'
194+
'powershell'
195+
'skopeo'
196+
'snapd'
197+
'tmux'
198+
)
199+
200+
if isX86; then
201+
packages+=(
202+
'google-chrome-stable'
203+
'google-cloud-cli'
204+
)
205+
fi
206+
207+
sudo apt-get purge -y --autoremove --fix-missing "${packages[@]}"
97208

209+
echo "=> apt-get autoremove"
98210
sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed"
99211
sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed"
100212
}
101213

102-
# Remove Docker images
103-
cleanDocker() {
104-
echo "Removing the following docker images:"
105-
sudo docker image ls
106-
echo "Removing docker images..."
107-
sudo docker image prune --all --force || true
108-
}
109-
110214
# Remove Swap storage
111215
cleanSwap() {
112216
sudo swapoff -a || true
113217
sudo rm -rf /mnt/swapfile || true
114218
free -h
115219
}
116220

117-
# Display initial disk space stats
118-
119-
AVAILABLE_INITIAL=$(getAvailableSpace)
221+
removePythonPackages() {
222+
local packages=(
223+
)
120224

121-
printDF "BEFORE CLEAN-UP:"
122-
echo ""
225+
if isX86; then
226+
packages+=(
227+
'ansible-core'
228+
)
229+
fi
123230

124-
removeDir /usr/local/lib/android
125-
removeDir /usr/share/dotnet
231+
for p in "${packages[@]}"; do
232+
sudo pipx uninstall "$p"
233+
done
234+
}
126235

127-
# Haskell runtime
128-
removeDir /opt/ghc
129-
removeDir /usr/local/.ghcup
236+
main() {
237+
printDF "BEFORE CLEAN-UP:"
130238

131-
execAndMeasureSpaceChange cleanPackages "Large misc. packages"
132-
execAndMeasureSpaceChange cleanDocker "Docker images"
133-
execAndMeasureSpaceChange cleanSwap "Swap storage"
239+
execAndMeasure "Unused packages" cleanPackages
240+
execAndMeasure "Swap storage" cleanSwap
241+
execAndMeasure "Node modules" removeNodeModules
242+
execAndMeasure "Python Packages" removePythonPackages
134243

135-
# Output saved space statistic
136-
echo ""
137-
printDF "AFTER CLEAN-UP:"
244+
removeUnusedDirsAndFiles
138245

139-
echo ""
140-
echo ""
246+
printDF "AFTER CLEAN-UP:"
247+
}
141248

142-
printSavedSpace "$AVAILABLE_INITIAL" "Total saved"
249+
execAndMeasure "Total" main

0 commit comments

Comments
 (0)