build: auto-set GitHub runner make job count

Dynamically determine an appropriate number of make jobs for the GitHub runner container, based upon the number of available CPU cores and RAM, using the rule that each job requires a dedicated core and 2.25GiB of RAM.
monero-project · Jan 16, 2025 · aefe0d2 · aefe0d2
1 parent 2e8a128
commit aefe0d2
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 9 deletions.
diff --git a/.github/actions/set-make-job-count/action.yml b/.github/actions/set-make-job-count/action.yml
@@ -0,0 +1,22 @@
+name: 'set-make-job-count'
+description: 'Set the MAKE_JOB_COUNT environment variable to a value suitable for the host runner'
+runs:
+  using: "composite"
+  steps:
+    # Each job runner requires 2.25 GiB (i.e. 1024 * 9/4 MiB) memory and
+    # a dedicated logical CPU core
+    - name: set-jobs-macOS
+      if: runner.os == 'macOS'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(sysctl -n hw.memsize) * 4 / (1073741824 * 9) )) $(sysctl -n hw.logicalcpu) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: bash
+    - name: set-jobs-windows
+      if: runner.os == 'Windows'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: msys2 {0}
+    - name: set-jobs-linux
+      if: runner.os == 'Linux'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: bash
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -13,8 +13,7 @@ on:
 # The below variables reduce repetitions across similar targets
 env:
   REMOVE_BUNDLED_PACKAGES : sudo rm -rf /usr/local
-  BUILD_DEFAULT_LINUX: |
-        cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build --parallel 4
+  BUILD_DEFAULT_LINUX: 'cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build'
   APT_INSTALL_LINUX: 'apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler ccache git'
   APT_SET_CONF: |
         tee -a /etc/apt/apt.conf.d/80-custom << EOF
@@ -41,14 +40,15 @@ jobs:
         path: /Users/runner/Library/Caches/ccache
         key: ccache-${{ runner.os }}-build-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-
+    - uses: ./.github/actions/set-make-job-count
     - name: install dependencies
       run: |
         HOMEBREW_NO_AUTO_UPDATE=1 brew install [email protected] hidapi openssl zmq miniupnpc expat libunwind-headers protobuf@21 ccache
         brew link protobuf@21 [email protected]
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make -j3
+        make -j${{env.MAKE_JOB_COUNT}}
 
   build-windows:
     runs-on: windows-latest
@@ -78,10 +78,11 @@ jobs:
         curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-libs-1.86.0-7-any.pkg.tar.zst
         echo "4cb1d1066fffa6a5788b212ccb920c6d8cc93a8ecbbc633565bfc9b2ebc6feb5 mingw-w64-x86_64-boost-libs-1.86.0-7-any.pkg.tar.zst" | sha256sum -c
         pacman --noconfirm -U mingw-w64-x86_64-boost-1.86.0-7-any.pkg.tar.zst mingw-w64-x86_64-boost-libs-1.86.0-7-any.pkg.tar.zst
+    - uses: ./.github/actions/set-make-job-count
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make release-static-win64 -j4
+        make release-static-win64 -j${{env.MAKE_JOB_COUNT}}
 
   build-debian:
     name: 'build-debian (debian-10)'
@@ -102,8 +103,9 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+      - uses: ./.github/actions/set-make-job-count
       - name: build
-        run: ${{env.BUILD_DEFAULT_LINUX}}
+        run: ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
 
 # See the OS labels and monitor deprecations here:
 # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
@@ -125,6 +127,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-${{ matrix.os }}
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -136,7 +139,7 @@ jobs:
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        ${{env.BUILD_DEFAULT_LINUX}}
+        ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
 
   libwallet-ubuntu:
     runs-on: ubuntu-20.04
@@ -151,6 +154,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-libwallet-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-libwallet-
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -163,7 +167,7 @@ jobs:
       run: |
         ${{env.CCACHE_SETTINGS}}
         cmake .
-        make wallet_api -j4
+        make wallet_api -j${{env.MAKE_JOB_COUNT}}
 
   test-ubuntu:
     needs: build-ubuntu
@@ -180,6 +184,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-ubuntu-latest
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -196,7 +201,7 @@ jobs:
         DNS_PUBLIC: tcp://9.9.9.9
       run: |
         ${{env.CCACHE_SETTINGS}}
-        ${{env.BUILD_DEFAULT_LINUX}}
+        ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
         cmake --build build --target test
 
 # ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere.

diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml
@@ -80,6 +80,7 @@ jobs:
         restore-keys: |
           depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
           depends-${{ matrix.toolchain.host }}-
+    - uses: ./.github/actions/set-make-job-count
     - name: set apt conf
       run: ${{env.APT_SET_CONF}}
     - name: install dependencies
@@ -92,7 +93,7 @@ jobs:
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make depends target=${{ matrix.toolchain.host }} -j4
+        make depends target=${{ matrix.toolchain.host }} -j${{env.MAKE_JOB_COUNT}}
     - uses: actions/upload-artifact@v4
       if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin' || matrix.toolchain.host == 'aarch64-apple-darwin' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }}
       with: