build: auto-set GitHub runner make job count

Dynamically determine an appropriate number of make jobs for the GitHub runner container, based upon the number of available CPU cores and RAM, using the rule that each job requires a dedicated core and 2.25GiB of RAM.
monero-project · Jan 16, 2025 · 1ce5d3a · 1ce5d3a
1 parent 1179269
commit 1ce5d3a
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 9 deletions.
diff --git a/.github/actions/set-make-job-count/action.yml b/.github/actions/set-make-job-count/action.yml
@@ -0,0 +1,22 @@
+name: 'set-make-job-count'
+description: 'Set the MAKE_JOB_COUNT environment variable to a value suitable for the host runner'
+runs:
+  using: "composite"
+  steps:
+    # Each job runner requires 2.25 GiB (i.e. 1024 * 9/4 MiB) memory and
+    # a dedicated logical CPU core
+    - name: set-jobs-macOS
+      if: runner.os == 'macOS'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(sysctl -n hw.memsize) * 4 / (1073741824 * 9) )) $(sysctl -n hw.logicalcpu) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: bash
+    - name: set-jobs-windows
+      if: runner.os == 'Windows'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: msys2 {0}
+    - name: set-jobs-linux
+      if: runner.os == 'Linux'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: bash
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -13,8 +13,7 @@ on:
 # The below variables reduce repetitions across similar targets
 env:
   REMOVE_BUNDLED_PACKAGES : sudo rm -rf /usr/local
-  BUILD_DEFAULT_LINUX: |
-        cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build --parallel 4
+  BUILD_DEFAULT_LINUX: 'cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build'
   APT_INSTALL_LINUX: 'apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler ccache git'
   APT_SET_CONF: |
         tee -a /etc/apt/apt.conf.d/80-custom << EOF
@@ -42,14 +41,15 @@ jobs:
         path: /Users/runner/Library/Caches/ccache
         key: ccache-${{ runner.os }}-build-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-
+    - uses: ./.github/actions/set-make-job-count
     - name: install dependencies
       run: |
         HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq miniupnpc expat libunwind-headers protobuf@21 ccache
         brew link protobuf@21 boost
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make -j3
+        make -j${{env.MAKE_JOB_COUNT}}
 
   build-windows:
     name: 'Windows (MSYS2)'
@@ -76,7 +76,7 @@ jobs:
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make release-static-win64 -j4
+        make release-static-win64 -j${{env.MAKE_JOB_COUNT}}
 
   build-debian:
     name: 'Debian 10'
@@ -97,8 +97,9 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+      - uses: ./.github/actions/set-make-job-count
       - name: build
-        run: ${{env.BUILD_DEFAULT_LINUX}}
+        run: ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
 
 # See the OS labels and monitor deprecations here:
 # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
@@ -125,6 +126,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-${{ matrix.os }}
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -136,7 +138,7 @@ jobs:
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        ${{env.BUILD_DEFAULT_LINUX}}
+        ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
 
   libwallet-ubuntu:
     name: "Ubuntu 20.04 (libwallet)"
@@ -152,6 +154,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-libwallet-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-libwallet-
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -164,7 +167,7 @@ jobs:
       run: |
         ${{env.CCACHE_SETTINGS}}
         cmake .
-        make wallet_api -j4
+        make wallet_api -j${{env.MAKE_JOB_COUNT}}
 
   test-ubuntu:
     name: "Ubuntu 20.04 (tests)"
@@ -182,6 +185,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-ubuntu-latest
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -198,7 +202,7 @@ jobs:
         DNS_PUBLIC: tcp://9.9.9.9
       run: |
         ${{env.CCACHE_SETTINGS}}
-        ${{env.BUILD_DEFAULT_LINUX}}
+        ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
         cmake --build build --target test
 
 # ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere.

diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml
@@ -79,6 +79,7 @@ jobs:
         restore-keys: |
           depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
           depends-${{ matrix.toolchain.host }}-
+    - uses: ./.github/actions/set-make-job-count
     - name: set apt conf
       run: ${{env.APT_SET_CONF}}
     - name: install dependencies
@@ -91,7 +92,7 @@ jobs:
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make depends target=${{ matrix.toolchain.host }} -j4
+        make depends target=${{ matrix.toolchain.host }} -j${{env.MAKE_JOB_COUNT}}
     - uses: actions/upload-artifact@v4
       if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin' || matrix.toolchain.host == 'aarch64-apple-darwin' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }}
       with: