Skip to content

Commit

Permalink
Run kustomize and kubectl sequentially, without piping (#119)
Browse files Browse the repository at this point in the history
This specifically fixes a bug where `ioutil.ReadAll` could block indefinitely
reading from kustomize's stderr pipe when kubectl exited before kustomize.

More generally, removing piping altogether reduces the complexity and
eliminates the distinct possibility of bugs like this being introduced again.

From my testing in exp-1-aws, where runs have been taking ~366 seconds, this
change adds about 4 seconds to the overall run latency. I think that's more than
acceptable.
  • Loading branch information
ribbybibby authored May 11, 2020
1 parent 16bdd56 commit dbbc2bc
Showing 1 changed file with 12 additions and 35 deletions.
47 changes: 12 additions & 35 deletions kubectl/client.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
package kubectl

import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os/exec"
"strings"

"github.com/utilitywarehouse/kube-applier/metrics"
)
Expand Down Expand Up @@ -48,51 +46,30 @@ func (c *Client) Apply(path, namespace string, dryRun, kustomize bool, pruneWhit
}

kubectlCmd := exec.Command(args[0], args[1:]...)
cmdStr := kubectlCmd.String()

var cmdStr string
var kustomizeStderr io.ReadCloser
if kustomize {
cmdStr = "kustomize build " + path + " | " + strings.Join(args, " ")
var kustomizeStdout, kustomizeStderr bytes.Buffer

kustomizeCmd := exec.Command("kustomize", "build", path)
kustomizeStdout, err := kustomizeCmd.StdoutPipe()
if err != nil {
return cmdStr, "", err
}
kubectlCmd.Stdin = kustomizeStdout
kustomizeStderr, err = kustomizeCmd.StderrPipe()
if err != nil {
return cmdStr, "", err
}
kustomizeCmd.Stdout = &kustomizeStdout
kustomizeCmd.Stderr = &kustomizeStderr

err = kustomizeCmd.Start()
err := kustomizeCmd.Run()
if err != nil {
fmt.Printf("%s", err)
return cmdStr, "", err
return kustomizeCmd.String(), kustomizeStderr.String(), err
}
defer func() {
io.Copy(ioutil.Discard, kustomizeStdout)
io.Copy(ioutil.Discard, kustomizeStderr)
err = kustomizeCmd.Wait()
if err != nil {
fmt.Printf("%s", err)
}
}()
} else {
cmdStr = strings.Join(args, " ")

kubectlCmd.Stdin = &kustomizeStdout
cmdStr = kustomizeCmd.String() + " | " + cmdStr
}

out, err := kubectlCmd.CombinedOutput()
if err != nil {
if e, ok := err.(*exec.ExitError); ok {
c.Metrics.UpdateKubectlExitCodeCount(namespace, e.ExitCode())
}
var str strings.Builder
if kustomizeStderr != nil {
kustomizeErr, _ := ioutil.ReadAll(kustomizeStderr)
str.WriteString(string(kustomizeErr))
}
str.WriteString(string(out))
return cmdStr, str.String(), err
return cmdStr, string(out), err
}
c.Metrics.UpdateKubectlExitCodeCount(path, 0)

Expand Down

0 comments on commit dbbc2bc

Please sign in to comment.