[experiment] ci: free more space in ubuntu 24 free runners

.github/workflows/ci.yml

@@ -99,6 +99,10 @@ jobs:
       - name: disable git crlf conversion
         run: git config --global core.autocrlf false
 
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
       - name: checkout the source code
         uses: actions/checkout@v4
         with:

src/ci/github-actions/jobs.yml

@@ -10,11 +10,6 @@ runners:
     free_disk: true
     <<: *base-job
 
-  # Large runner used mainly for its bigger disk capacity
-  - &job-linux-4c-largedisk
-    os: ubuntu-22.04-4core-16gb
-    <<: *base-job
-
   - &job-linux-8c
     os: ubuntu-22.04-8core-32gb
     <<: *base-job
@@ -182,7 +177,7 @@ auto:
     <<: *job-linux-4c
 
   - name: dist-powerpc64le-linux
-    <<: *job-linux-4c-largedisk
+    <<: *job-linux-4c
 
   - name: dist-riscv64-linux
     <<: *job-linux-4c
@@ -295,7 +290,7 @@ auto:
   - name: x86_64-gnu-debug
     # This seems to be needed because a full stage 2 build + run-make tests
     # overwhelms the storage capacity of the standard 4c runner.
-    <<: *job-linux-4c-largedisk
+    <<: *job-linux-4c
 
   - name: x86_64-gnu-distcheck
     <<: *job-linux-8c

src/ci/scripts/free-disk-space.sh

@@ -1,9 +1,15 @@
 #!/bin/bash
 set -euo pipefail
 
-# Free disk space on Linux GitHub action runners
+# Free disk space on Linux GitHub action runners.
 # Script inspired by https://github.com./jlumbroso/free-disk-space
 
+# When updating to a new ubuntu version (e.g. from ubuntu-24.04):
+# - Check that there are no docker images preinstalled with `docker image ls`
+# - Check that there are no big packages preinstalled that we aren't using
+# - Check that all directores we are removing are still present (look at the warnings)
+
+# Check if the architecture is x86.
 isX86() {
     local arch
     arch=$(uname -m)
@@ -14,49 +20,55 @@ isX86() {
     fi
 }
 
-# print a line of the specified character
+# Print a line of the specified character.
 printSeparationLine() {
     for ((i = 0; i < 80; i++)); do
         printf "%s" "$1"
     done
     printf "\n"
 }
 
-# compute available space
+# Compute available space.
 # REF: https://unix.stackexchange.com/a/42049/60849
 # REF: https://stackoverflow.com/a/450821/408734
 getAvailableSpace() {
     df -a | awk 'NR > 1 {avail+=$4} END {print avail}'
 }
 
-# make Kb human readable (assume the input is Kb)
+# Make Kb human readable (assume the input is Kb).
 # REF: https://unix.stackexchange.com/a/44087/60849
 formatByteCount() {
     numfmt --to=iec-i --suffix=B --padding=7 "$1"'000'
 }
 
-# macro to output saved space
-printSavedSpace() {
-    # Disk space before the operation
-    local before=${1}
-    local title=${2:-}
+# Execute a task, printing how much space was saved and how long it took to run the task
+execAndMeasure() {
+    local task_name=${1}
+
+    local start
+    start=$(date +%s)
+
+    local before
+    before=$(getAvailableSpace)
+
+    # Run the task. Skip the first argument because it's the task name.
+    "${@:2}"
+
+    local end
+    end=$(date +%s)
 
     local after
     after=$(getAvailableSpace)
+
+    # How much space was saved.
     local saved=$((after - before))
+    # How long the task took.
+    local seconds_taken=$((end - start))
 
-    echo ""
-    printSeparationLine "*"
-    if [ -n "${title}" ]; then
-        echo "=> ${title}: Saved $(formatByteCount "$saved")"
-    else
-        echo "=> Saved $(formatByteCount "$saved")"
-    fi
-    printSeparationLine "*"
-    echo ""
+    echo "==> ${task_name}: Saved $(formatByteCount "$saved") in $seconds_taken seconds"
 }
 
-# macro to print output of df with caption
+# Print output of df with caption. It shows information about disk space.
 printDF() {
     local caption=${1}
 
@@ -67,110 +79,166 @@ printDF() {
     echo ""
     df -h
     printSeparationLine "="
+    echo ""
 }
 
-removeDir() {
-    dir=${1}
-
-    local before
-    if [ ! -d "$dir" ]; then
-        echo "::warning::Directory $dir does not exist, skipping."
-    else
-        before=$(getAvailableSpace)
-        sudo rm -rf "$dir"
-        printSavedSpace "$before" "Removed $dir"
-    fi
-}
-
-removeUnusedDirectories() {
-    local dirs_to_remove=(
+removeUnusedDirsAndFiles() {
+    local to_remove=(
+        "/etc/mysql"
+        "/usr/local/aws-sam-cli"
+        "/usr/local/doc/cmake"
+        "/usr/local/julia"*
         "/usr/local/lib/android"
-        "/usr/share/dotnet"
+        "/usr/local/share/chromedriver-"*
+        "/usr/local/share/chromium"
+        "/usr/local/share/cmake-"*
+        "/usr/local/share/edge_driver"
+        "/usr/local/share/gecko_driver"
+        "/usr/local/share/icons"
+        "/usr/local/share/vim"
+        "/usr/local/share/emacs"
+        "/usr/local/share/powershell"
+        "/usr/local/share/vcpkg"
+        "/usr/share/apache-maven-"*
+        "/usr/share/gradle-"*
+        "/usr/share/java"
+        "/usr/share/kotlinc"
+        "/usr/share/miniconda"
+        "/usr/share/php"
+        "/usr/share/ri"
+        "/usr/share/swift"
+
+        # binaries
+        "/usr/local/bin/azcopy"
+        "/usr/local/bin/bicep"
+        "/usr/local/bin/ccmake"
+        "/usr/local/bin/cmake-"*
+        "/usr/local/bin/cmake"
+        "/usr/local/bin/cpack"
+        "/usr/local/bin/ctest"
+        "/usr/local/bin/helm"
+        "/usr/local/bin/kind"
+        "/usr/local/bin/kustomize"
+        "/usr/local/bin/minikube"
+        "/usr/local/bin/packer"
+        "/usr/local/bin/phpunit"
+        "/usr/local/bin/pulumi-"*
+        "/usr/local/bin/pulumi"
+        "/usr/local/bin/stack"
 
         # Haskell runtime
         "/usr/local/.ghcup"
+
+        # Azure
+        "/opt/az"
+        "/usr/share/az_"*
+
+        # Environemnt variable set by GitHub Actions
+        "$AGENT_TOOLSDIRECTORY"
     )
 
-    for dir in "${dirs_to_remove[@]}"; do
-        removeDir "$dir"
+    local existing=()
+    for element in "${to_remove[@]}"; do
+        if [ ! -e "$element" ]; then
+            echo "::warning::Directory or file $element does not exist, skipping."
+        else
+            existing+=("$element")
+        fi
     done
-}
-
-execAndMeasureSpaceChange() {
-    local operation=${1} # Function to execute
-    local title=${2}
 
-    local before
-    before=$(getAvailableSpace)
-    $operation
+    execAndMeasure "Removed unused directories" sudo rm -rf "${existing[@]}"
+}
 
-    printSavedSpace "$before" "$title"
+removeNodeModules() {
+    sudo npm uninstall -g \
+        "@bazel/bazelisk" \
+        "grunt"           \
+        "gulp"            \
+        "lerna"           \
+        "n"               \
+        "newman"          \
+        "parcel"          \
+        "typescript"      \
+        "webpack-cli"     \
+        "webpack"         \
+        "yarn"
 }
 
-# Remove large packages
-# REF: https://github.com./apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
+# Remove unused packages.
 cleanPackages() {
     local packages=(
+        '.*-icon-theme$'
         '^aspnetcore-.*'
         '^dotnet-.*'
+        '^java-*'
+        '^libllvm.*'
         '^llvm-.*'
-        '^mongodb-.*'
+        '^mercurial.*'
         '^mysql-.*'
+        '^vim.*'
+        '^fonts-.*'
         'azure-cli'
+        'buildah'
+        'cpp-13'
         'firefox'
+        'gcc-12'
+        'gcc-13'
+        'gcc-14'
+        'gcc'
+        'g++-14'
+        'gfortran-14'
+        'groff-base'
+        'kubectl'
         'libgl1-mesa-dri'
-        'mono-devel'
+        'microsoft-edge-stable'
         'php.*'
+        'podman'
+        'powershell'
+        'skopeo'
+        'snapd'
+        'tmux'
     )
 
     if isX86; then
         packages+=(
             'google-chrome-stable'
             'google-cloud-cli'
-            'google-cloud-sdk'
-            'powershell'
         )
     fi
 
-    sudo apt-get -qq remove -y --fix-missing "${packages[@]}"
+    sudo apt-get purge -y --autoremove --fix-missing "${packages[@]}"
 
+    echo "=> apt-get autoremove"
     sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed"
     sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed"
 }
 
-# Remove Docker images
-cleanDocker() {
-    echo "=> Removing the following docker images:"
-    sudo docker image ls
-    echo "=> Removing docker images..."
-    sudo docker image prune --all --force || true
-}
-
-# Remove Swap storage
-cleanSwap() {
-    sudo swapoff -a || true
-    sudo rm -rf /mnt/swapfile || true
-    free -h
-}
-
-# Display initial disk space stats
+removePythonPackages() {
+    local packages=(
+    )
 
-AVAILABLE_INITIAL=$(getAvailableSpace)
+    if isX86; then
+        packages+=(
+            'ansible-core'
+        )
+    fi
 
-printDF "BEFORE CLEAN-UP:"
-echo ""
+    for p in "${packages[@]}"; do
+        sudo pipx uninstall "$p"
+    done
+}
 
-execAndMeasureSpaceChange cleanPackages "Unused packages"
-execAndMeasureSpaceChange cleanDocker "Docker images"
-execAndMeasureSpaceChange cleanSwap "Swap storage"
+main() {
+    printDF "BEFORE CLEAN-UP:"
 
-removeUnusedDirectories
+    execAndMeasure "Unused packages" cleanPackages
+    execAndMeasure "Unused Python Packages" removePythonPackages
+    execAndMeasure "Unused Node modules" removeNodeModules
 
-# Output saved space statistic
-echo ""
-printDF "AFTER CLEAN-UP:"
+    removeUnusedDirsAndFiles
 
-echo ""
-echo ""
+    # Output saved space statistic
+    printDF "AFTER CLEAN-UP:"
+}
 
-printSavedSpace "$AVAILABLE_INITIAL" "Total saved"
+execAndMeasure "Total" main