From 1fecb738aad0a9c114ac34228b744ee6d0b3a84d Mon Sep 17 00:00:00 2001
From: Salvatore Mesoraca <salvatore.mesoraca@aiven.io>
Date: Wed, 4 Sep 2024 17:51:18 +0200
Subject: [PATCH] ci: cache docker layers and avoid re-downloading them for
 each job

Sometimes the CI fails because Docker doesn't respond.
It could be because we download many docker images in every job
and we have 58 jobs now.
With this commit, the images layers, will be downloaded once at the beginning
and then re-used by all subsequent jobs.
This may even speedup the CI a bit or make it slower if the cache is not
very fast.
But hopefully we won't have spurious CI failures because of Docker anymore.

Signed-off-by: Salvatore Mesoraca <salvatore.mesoraca@aiven.io>
---
 .github/workflows/integration.yaml | 33 ++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index df15684b..ec4c7e5b 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -50,8 +50,21 @@ jobs:
            pip install -r dev_requirements.txt
            invoke linters
 
+   populate-cache:
+     runs-on: ubuntu-latest
+     timeout-minutes: 60
+     name: Update docker cache
+     steps:
+       - name: Cache docker images
+         uses: actions/cache@v3
+         with:
+           path: ./custom-cache/
+       - name: Update Cache
+         run: docker compose --profile all build -d && docker save $(docker images -q) -o ./custom-cache/all.tar
+
    run-tests:
      runs-on: ubuntu-latest
+     needs: [populate-cache]
      timeout-minutes: 60
      strategy:
        max-parallel: 15
@@ -70,6 +83,16 @@ jobs:
          with:
            python-version: ${{ matrix.python-version }}
            cache: 'pip'
+
+       - name: Cache docker images
+         id: custom-cache
+         uses: actions/cache@v3
+         with:
+           path: ./custom-cache/
+           fail-on-cache-miss: true
+       - name: Use Cache
+         run: docker image load -i ./custom-cache/all.tar
+
        - name: run tests
          run: |
            pip install -U setuptools wheel
@@ -115,6 +138,16 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
+
+      - name: Cache docker images
+        id: custom-cache
+        uses: actions/cache@v3
+        with:
+          path: ./custom-cache/
+          fail-on-cache-miss: true
+      - name: Use Cache
+        run: docker image load -i ./custom-cache/all.tar
+
       - name: Run installed unit tests
         run: |
           bash .github/workflows/install_and_test.sh ${{ matrix.extension }}