From d7f50dfe6a361557968a4a01278d8ea0d96e227e Mon Sep 17 00:00:00 2001
From: tserg <8017125+tserg@users.noreply.github.com>
Date: Tue, 21 Jan 2025 00:14:26 +0800
Subject: [PATCH] feat[ci]: use `coverage combine` to reduce codecov uploads
 (#4452)

This commit updates the CI to combine code coverage to a single file
and perform just a single upload after all tests finish (rather than
each test run uploading its own coverage report). This should reduce
failures / rate limiting on the codecov app, and also prevent the
codecov app from producing an inaccurate coverage report before all
the tests finish.

references:
- https://coverage.readthedocs.io/en/7.6.10/cmd.html#cmd-combine
- https://coverage.readthedocs.io/en/7.6.10/cmd.html#re-mapping-paths
- https://coverage.readthedocs.io/en/7.6.10/config.html#config-run-relative-files

---------

Co-authored-by: Charles Cooper <cooper.charles.m@gmail.com>
---
 .github/workflows/test.yml | 61 ++++++++++++++++++++++++++++++--------
 setup.cfg                  | 12 ++++++++
 2 files changed, 61 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index aba60ba391..c9705d5e87 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -149,16 +149,17 @@ jobs:
         --evm-backend ${{ matrix.evm-backend || 'revm' }}
         ${{ matrix.debug && '--enable-compiler-debug-mode' || '' }}
         ${{ matrix.experimental-codegen && '--experimental-codegen' || '' }}
-        --cov-branch
-        --cov-report xml:coverage.xml
+        --cov-config=setup.cfg
         --cov=vyper
         tests/
 
-    - name: Upload Coverage
-      uses: codecov/codecov-action@v5
+    - name: Upload coverage artifact
+      uses: actions/upload-artifact@v4
       with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: ./coverage.xml
+        name: coverage-files-${{ github.job }}-${{ strategy.job-index }}
+        include-hidden-files: true
+        path: .coverage
+        if-no-files-found: error
 
   core-tests-success:
     if: always()
@@ -209,16 +210,17 @@ jobs:
           --splits 120 \
           --group ${{ matrix.group }} \
           --splitting-algorithm least_duration \
-          --cov-branch \
-          --cov-report xml:coverage.xml \
+          --cov-config=setup.cfg \
           --cov=vyper \
           tests/
 
-    - name: Upload Coverage
-      uses: codecov/codecov-action@v5
+    - name: Upload coverage artifact
+      uses: actions/upload-artifact@v4
       with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: ./coverage.xml
+        name: coverage-files-${{ github.job }}-${{ strategy.job-index }}
+        include-hidden-files: true
+        path: .coverage
+        if-no-files-found: error
 
   slow-tests-success:
     if: always()
@@ -231,3 +233,38 @@ jobs:
       - name: Check slow tests all succeeded
         if: ${{ needs.fuzzing.result != 'success' }}
         run: exit 1
+
+  consolidate-coverage:
+    # Consolidate code coverage using `coverage combine` and upload
+    # to the codecov app
+    runs-on: ubuntu-latest
+    needs: [tests, fuzzing]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+        cache: "pip"
+  
+    - name: Install coverage
+      run: pip install coverage
+
+    - name: Download coverage artifacts
+      uses: actions/download-artifact@v4
+      with:
+        pattern: coverage-files-*
+        path: coverage-files
+
+    - name: Combine coverage
+      run: |
+        coverage combine coverage-files/**/.coverage
+        coverage xml
+
+    - name: Upload Coverage
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml
diff --git a/setup.cfg b/setup.cfg
index 5998961ee8..4cce85034d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -33,3 +33,15 @@ markers =
 	fuzzing: Run Hypothesis fuzz test suite (deselect with '-m "not fuzzing"')
 	requires_evm_version(version): Mark tests that require at least a specific EVM version and would throw `EvmVersionException` otherwise
 	venom_xfail: mark a test case as a regression (expected to fail) under the venom pipeline
+
+
+[coverage:run]
+branch = True
+source = vyper
+
+# this is not available on the CI step that performs `coverage combine`
+omit = vyper/version.py
+
+# allow `coverage combine` to combine reports from heterogeneous OSes.
+# (mainly important for consolidating coverage reports in the CI).
+relative_files = True