Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: optimize Terraform based change detection #1913

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

chore: optimize Terraform based change detection #1913

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b1c2bde
refactor: improve Terraform module change detection.
i4ki Oct 10, 2024
c9d7d98
Merge branch 'main' of github.com:terramate-io/terramate into i4k-cac…
i4ki Oct 10, 2024
0b87d61
chore: optimize Terraform module detection.
i4ki Oct 10, 2024
b47f832
Merge branch 'main' into i4k-cache-terraform-change-detection
i4ki Oct 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 113 additions & 64 deletions stack/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@ type (
git *git.Git

cache struct {
stacks []Entry
changedFiles map[string][]string // gitBaseRef -> changed files
stacks []Entry
changedFiles map[string][]string // gitBaseRef -> changed files
changedModules map[string]isChangedCache
}
}

Expand All @@ -43,6 +44,11 @@ type (
UntrackedChanges *bool
}

isChangedCache struct {
isChanged bool
reason string
}

// Report is the report of project's stacks and the result of its default checks.
Report struct {
Stacks []Entry
Expand Down Expand Up @@ -73,6 +79,7 @@ func NewManager(root *config.Root) *Manager {
root: root,
}
m.cache.changedFiles = make(map[string][]string)
m.cache.changedModules = make(map[string]isChangedCache)
return m
}

Expand All @@ -83,6 +90,7 @@ func NewGitAwareManager(root *config.Root, git *git.Git) *Manager {
git: git,
}
m.cache.changedFiles = make(map[string][]string)
m.cache.changedModules = make(map[string]isChangedCache)
return m
}

Expand Down Expand Up @@ -292,10 +300,10 @@ func (m *Manager) ListChanged(cfg ChangeConfig) (*Report, error) {
}
}

rangeStacks:
for _, stackEntry := range allstacks {
stack := stackEntry.Stack
if _, ok := stackSet[stack.Dir]; ok {
// stack already changed.
continue
}

Expand All @@ -313,50 +321,25 @@ rangeStacks:
changed,
),
}
continue rangeStacks
continue
}

// Terraform module change detection
err := m.filesApply(stack.Dir, func(fname string) error {
if path.Ext(fname) != ".tf" {
return nil
}

tfpath := filepath.Join(stack.HostDir(m.root), fname)

modules, err := tf.ParseModules(tfpath)
if err != nil {
return errors.E(errListChanged, "parsing modules", err)
}

for _, mod := range modules {
changed, why, err := m.tfModuleChanged(mod, stack.HostDir(m.root), cfg.BaseRef, make(map[string]bool))
if err != nil {
return errors.E(errListChanged, err, "checking module %q", mod.Source)
}
mod, why, isChanged, err := m.checkIfStackTerraformModulesChanged(stack, cfg)
if err != nil {
return nil, errors.E(errListChanged, err)
}

if changed {
logger.Debug().
Stringer("stack", stack).
Str("configFile", tfpath).
Msg("Module changed.")

stack.IsChanged = true
stackSet[stack.Dir] = Entry{
Stack: stack,
Reason: fmt.Sprintf(
"stack changed because %q changed because %s",
mod.Source, why,
),
}
return nil
}
if isChanged {
stack.IsChanged = true
stackSet[stack.Dir] = Entry{
Stack: stack,
Reason: fmt.Sprintf(
"stack changed because %q changed because %s",
mod.Source, why,
),
}
return nil
})

if err != nil {
return nil, errors.E(errListChanged, "checking if Terraform module changes", err)
continue
}

// tgModulesMap is only populated if Terragrunt is enabled.
Expand All @@ -381,7 +364,7 @@ rangeStacks:
Stack: stack,
Reason: fmt.Sprintf("stack changed because module %q changed because %s", tgMod.Path, why),
}
continue rangeStacks
continue
}
}

Expand All @@ -402,6 +385,57 @@ rangeStacks:
}, nil
}

func (m *Manager) checkIfStackTerraformModulesChanged(stack *config.Stack, cfg ChangeConfig) (
changedModule tf.Module, reason string, isChanged bool, err error,
) {
logger := log.With().
Str("action", "checkIfStackTerraformModulesChanged").
Stringer("stack", stack.Dir).
Logger()

// Terraform module change detection
err = m.filesApply(stack.Dir, func(fname string) (bool, error) {
if path.Ext(fname) != ".tf" {
return false, nil
}

tfpath := filepath.Join(stack.HostDir(m.root), fname)

modules, err := tf.ParseModules(tfpath)
if err != nil {
return false, errors.E(errListChanged, "parsing modules", err)
}

stackHostDir := stack.HostDir(m.root)

for _, mod := range modules {
changed, why, err := m.tfModuleChanged(mod, stackHostDir, cfg.BaseRef)
if err != nil {
return false, errors.E(errListChanged, err, "checking module %q", mod.Source)
}

if changed {
logger.Debug().
Str("configFile", tfpath).
Msg("Module changed.")

isChanged = changed
reason = why
changedModule = mod

return true, nil
}
}
return false, nil
})

if err != nil {
return tf.Module{}, "", false, errors.E(errListChanged, "checking if Terraform module changes", err)
}

return changedModule, reason, isChanged, nil
}

func (m *Manager) allStacks() ([]Entry, error) {
var allstacks []Entry
if m.cache.stacks != nil {
Expand Down Expand Up @@ -506,7 +540,7 @@ func (m *Manager) AddWantedOf(scopeStacks config.List[*config.SortableStack]) (c
return selectedStacks, nil
}

func (m *Manager) filesApply(dir project.Path, apply func(fname string) error) (err error) {
func (m *Manager) filesApply(dir project.Path, apply func(fname string) (bool, error)) error {
var files []string

tree, skipped, ok := m.root.Lookup2(dir)
Expand Down Expand Up @@ -545,10 +579,13 @@ func (m *Manager) filesApply(dir project.Path, apply func(fname string) error) (
}

for _, fname := range files {
err := apply(fname)
stop, err := apply(fname)
if err != nil {
return errors.E(err, "applying operation to file %q", fname)
}
if stop {
break
}
}
return nil
}
Expand All @@ -558,21 +595,33 @@ func (m *Manager) filesApply(dir project.Path, apply func(fname string) error) (
// called recursively. The visited keep track of the modules already parsed to
// avoid infinite loops.
func (m *Manager) tfModuleChanged(
mod tf.Module, basedir string, gitBaseRef string, visited map[string]bool,
mod tf.Module, basedir string, gitBaseRef string,
) (changed bool, why string, err error) {
if _, ok := visited[mod.Source]; ok {
return false, "", nil
modAbsPath := filepath.Join(basedir, mod.Source)
modPath := project.PrjAbsPath(m.root.HostDir(), modAbsPath)
modPathStr := modPath.String()

cached, ok := m.cache.changedModules[modPath.String()]
if ok {
return cached.isChanged, cached.reason, nil
}

defer func() {
_, ok := m.cache.changedModules[modPathStr]
if !ok {
m.cache.changedModules[modPathStr] = isChangedCache{
isChanged: changed,
reason: why,
}
}
}()

if !mod.IsLocal() {
// if the source is a remote path (URL, VCS path, S3 bucket, etc) then
// we assume it's not changed.
return false, "", nil
}

modAbsPath := filepath.Join(basedir, mod.Source)
modPath := project.PrjAbsPath(m.root.HostDir(), modAbsPath)

st, err := os.Stat(modAbsPath)

// TODO(i4k): resolve symlinks
Expand All @@ -592,36 +641,33 @@ func (m *Manager) tfModuleChanged(
}
}

visited[mod.Source] = true

err = m.filesApply(modPath, func(fname string) error {
err = m.filesApply(modPath, func(fname string) (bool, error) {
if changed {
return nil
return true, nil
}
if path.Ext(fname) != ".tf" {
return nil
return false, nil
}

modules, err := tf.ParseModules(filepath.Join(modAbsPath, fname))
if err != nil {
return errors.E(err, "parsing module %q", mod.Source)
return false, errors.E(err, "parsing module %q", mod.Source)
}

for _, mod2 := range modules {
var reason string

changed, reason, err = m.tfModuleChanged(mod2, modAbsPath, gitBaseRef, visited)
changed, reason, err = m.tfModuleChanged(mod2, modAbsPath, gitBaseRef)
if err != nil {
return err
return false, err
}

if changed {
why = fmt.Sprintf("%s%s changed because %s ", why, mod.Source, reason)
return nil
why = fmt.Sprintf("%s%s changed because %s ", why, mod2.Source, reason)
return true, nil
}
}

return nil
return false, nil
})

if err != nil {
Expand Down Expand Up @@ -650,7 +696,7 @@ func (m *Manager) tgModuleChanged(
) (changed bool, why string, err error) {
tfMod := tf.Module{Source: tgMod.Source}
if tfMod.IsLocal() {
changed, why, err := m.tfModuleChanged(tfMod, project.AbsPath(m.root.HostDir(), tgMod.Path.String()), gitBaseRef, make(map[string]bool))
changed, why, err := m.tfModuleChanged(tfMod, project.AbsPath(m.root.HostDir(), tgMod.Path.String()), gitBaseRef)
if err != nil {
return false, "", errors.E(errListChanged, err, "checking if Terraform module changes (in Terragrunt context)")
}
Expand Down Expand Up @@ -748,6 +794,9 @@ func (m *Manager) listChangedFiles(dir string, gitBaseRef string) ([]string, err
}

func hasChangedWatchedFiles(stack *config.Stack, changedFiles []string) (project.Path, bool) {
if len(changedFiles) == 0 {
return project.Path{}, false
}
for _, watchFile := range stack.Watch {
for _, file := range changedFiles {
if file == watchFile.String()[1:] { // project paths
Expand Down
3 changes: 0 additions & 3 deletions tf/mod_source_parse.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,6 @@ func ParseSource(modsource string) (Source, error) {
pathstr := path.Join(strings.Replace(u.Host, ":", "-", -1), u.Path)
pathstr = strings.TrimSuffix(pathstr, ".git")

if err != nil {
return Source{}, err
}
ref := u.Query().Get("ref")
u.RawQuery = ""
return Source{
Expand Down
Loading