Mitigate DiskFull failures in subsequent tasks after docker image fai…

…lures (#711)
microsoft · May 19, 2024 · dc47555 · dc47555
1 parent 4244ae7
commit dc47555
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 12 deletions.
diff --git a/src/Tes.Runner.Test/Docker/DockerExecutorTests.cs b/src/Tes.Runner.Test/Docker/DockerExecutorTests.cs
@@ -56,7 +56,7 @@ public async Task RunOnContainerAsync_DockerClientReturnsAuthNeeded_CallsContain
                 Assert.Fail(ex.Message);
             }
 
-            Assert.AreEqual(1, dockerImageMock.Invocations.Count);
+            Assert.AreEqual(2, dockerImageMock.Invocations.Count);
         }
 
         [DataTestMethod]
@@ -85,7 +85,7 @@ public async Task RunOnContainerAsync_DockerClientReturnsOtherError_DoesNotCallC
                 Assert.AreSame(exception, ex);
             }
 
-            Assert.AreEqual(1 + DockerExecutor.dockerPullRetryPolicyOptions.MaxRetryCount, dockerImageMock.Invocations.Count);
+            Assert.AreEqual(2 + DockerExecutor.dockerPullRetryPolicyOptions.MaxRetryCount, dockerImageMock.Invocations.Count);
         }
 
         //[DataTestMethod]

diff --git a/src/Tes.Runner/Docker/DockerExecutor.cs b/src/Tes.Runner/Docker/DockerExecutor.cs
@@ -85,21 +85,29 @@ public virtual async Task<ContainerExecutionResult> RunOnContainerAsync(Executio
 
             try
             {
-                await PullImageWithRetriesAsync(executionOptions.ImageName, executionOptions.Tag);
-            }
-            catch (DockerApiException e) when (IsAuthFailure(e))
-            {
-                var authConfig = await containerRegistryAuthorizationManager.TryGetAuthConfigForAzureContainerRegistryAsync(executionOptions.ImageName, executionOptions.Tag, executionOptions.RuntimeOptions);
-
-                if (authConfig is not null)
+                try
                 {
-                    await PullImageWithRetriesAsync(executionOptions.ImageName, executionOptions.Tag, authConfig);
+                    await PullImageWithRetriesAsync(executionOptions.ImageName, executionOptions.Tag);
                 }
-                else
+                catch (DockerApiException e) when (IsAuthFailure(e))
                 {
-                    throw;
+                    var authConfig = await containerRegistryAuthorizationManager.TryGetAuthConfigForAzureContainerRegistryAsync(executionOptions.ImageName, executionOptions.Tag, executionOptions.RuntimeOptions);
+
+                    if (authConfig is not null)
+                    {
+                        await PullImageWithRetriesAsync(executionOptions.ImageName, executionOptions.Tag, authConfig);
+                    }
+                    else
+                    {
+                        throw;
+                    }
                 }
             }
+            catch
+            {
+                await dockerClient.Images.PruneImagesAsync();
+                throw;
+            }
 
             await ConfigureNetworkAsync();