From 2a2ace5b992f22ba1b77cf3dcf0a344519a8653f Mon Sep 17 00:00:00 2001
From: Salvatore Mesoraca <salvatore.mesoraca@aiven.io>
Date: Wed, 4 Sep 2024 17:51:18 +0200
Subject: [PATCH] ci: cache docker layers and avoid re-downloading them for
 each job

Sometimes the CI fails because Docker doesn't respond.
It could be because we download many docker images in every
job and we have 58 jobs now.
With this commit, the image layers will be downloaded
during nightly builds and cached.

This seems to cut the CI running time in half from 33~ minutes to 15~.

Signed-off-by: Salvatore Mesoraca <salvatore.mesoraca@aiven.io>
---
 .github/workflows/integration.yaml | 42 ++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index df15684b..bdefbe18 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -50,8 +50,28 @@ jobs:
            pip install -r dev_requirements.txt
            invoke linters
 
+   populate-cache:
+     runs-on: ubuntu-latest
+     timeout-minutes: 60
+     name: Update docker cache
+     steps:
+       - uses: actions/checkout@v4
+       - name: Cache docker images
+         id: custom-cache
+         uses: actions/cache@v3
+         with:
+           path: ./custom-cache/
+           key: custom-cache
+       - if: ${{ steps.custom-cache.outputs.cache-hit != 'true' || github.event_name == 'schedule' }}
+         name: Update Cache
+         run: |
+           mkdir -p ./custom-cache/
+           docker compose --profile all build
+           docker save valkey-py-stunnel:latest valkey-py-cluster:latest valkey/valkey:latest -o ./custom-cache/all.tar
+
    run-tests:
      runs-on: ubuntu-latest
+     needs: [populate-cache]
      timeout-minutes: 60
      strategy:
        max-parallel: 15
@@ -70,6 +90,17 @@ jobs:
          with:
            python-version: ${{ matrix.python-version }}
            cache: 'pip'
+
+       - name: Cache docker images
+         id: custom-cache
+         uses: actions/cache@v3
+         with:
+           path: ./custom-cache/
+           fail-on-cache-miss: true
+           key: custom-cache
+       - name: Use Cache
+         run: docker image load -i ./custom-cache/all.tar
+
        - name: run tests
          run: |
            pip install -U setuptools wheel
@@ -115,6 +146,17 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
+
+      - name: Cache docker images
+        id: custom-cache
+        uses: actions/cache@v3
+        with:
+          path: ./custom-cache/
+          fail-on-cache-miss: true
+          key: custom-cache
+      - name: Use Cache
+        run: docker image load -i ./custom-cache/all.tar
+
       - name: Run installed unit tests
         run: |
           bash .github/workflows/install_and_test.sh ${{ matrix.extension }}