From ce61e7bf182c85b747cb8dcf10fc4ccfd15b62ec Mon Sep 17 00:00:00 2001 From: Nick Santos Date: Mon, 29 Jul 2019 11:18:22 -0400 Subject: [PATCH 01/10] ignore: improve the ephemeral temp file patterns [ch2663] (#1925) --- pkg/watch/ephemeral.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 pkg/watch/ephemeral.go diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go new file mode 100644 index 00000000..ae1ea96c --- /dev/null +++ b/pkg/watch/ephemeral.go @@ -0,0 +1,34 @@ +package ignore + +import ( + "github.com/windmilleng/tilt/internal/dockerignore" + "github.com/windmilleng/tilt/internal/model" +) + +// Filter out spurious changes that we don't want to rebuild on, like IDE +// temp/lock files. +// +// This isn't an ideal solution. In an ideal world, the user would put +// everything to ignore in their tiltignore/dockerignore files. This is a +// stop-gap so they don't have a terrible experience if those files aren't +// there or aren't in the right places. +// +// https://app.clubhouse.io/windmill/story/691/filter-out-ephemeral-file-changes +var ephemeralPathMatcher = initEphemeralPathMatcher() + +func initEphemeralPathMatcher() model.PathMatcher { + golandPatterns := []string{"**/*___jb_old___", "**/*___jb_tmp___"} + emacsPatterns := []string{"**/.#*"} + vimPatterns := []string{"**/4913", "**/*~", "**/.*.swp", "**/.*.swx"} + + allPatterns := []string{} + allPatterns = append(allPatterns, golandPatterns...) + allPatterns = append(allPatterns, emacsPatterns...) + allPatterns = append(allPatterns, vimPatterns...) + + matcher, err := dockerignore.NewDockerPatternMatcher("/", allPatterns) + if err != nil { + panic(err) + } + return matcher +} From 91a2bdd6dedc74a2d499031aeb88dde1adea9950 Mon Sep 17 00:00:00 2001 From: Dan Miller Date: Fri, 9 Aug 2019 12:52:31 -0400 Subject: [PATCH 02/10] model: move to pkg (#2024) --- pkg/watch/ephemeral.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index ae1ea96c..48d224de 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -2,7 +2,7 @@ package ignore import ( "github.com/windmilleng/tilt/internal/dockerignore" - "github.com/windmilleng/tilt/internal/model" + "github.com/windmilleng/tilt/pkg/model" ) // Filter out spurious changes that we don't want to rebuild on, like IDE From e3948f6bae74e968cf4f0e8b40b6e81f2ee93bc8 Mon Sep 17 00:00:00 2001 From: Maia McCormick Date: Thu, 15 Aug 2019 14:38:44 -0400 Subject: [PATCH 03/10] ignore: auto-ignore jetbrains .idea file (#2065) --- pkg/watch/ephemeral.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index 48d224de..e48300ba 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -17,7 +17,7 @@ import ( var ephemeralPathMatcher = initEphemeralPathMatcher() func initEphemeralPathMatcher() model.PathMatcher { - golandPatterns := []string{"**/*___jb_old___", "**/*___jb_tmp___"} + golandPatterns := []string{"**/*___jb_old___", "**/*___jb_tmp___", "**/.idea/**"} emacsPatterns := []string{"**/.#*"} vimPatterns := []string{"**/4913", "**/*~", "**/.*.swp", "**/.*.swx"} From 12916b75a22e878d529b87f6d1d78170290848a2 Mon Sep 17 00:00:00 2001 From: Matt Landis Date: Thu, 12 Sep 2019 11:57:39 -0400 Subject: [PATCH 04/10] tilt: ignore a few more vim swap files (#2190) --- pkg/watch/ephemeral.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index e48300ba..6b260481 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -19,7 +19,12 @@ var ephemeralPathMatcher = initEphemeralPathMatcher() func initEphemeralPathMatcher() model.PathMatcher { golandPatterns := []string{"**/*___jb_old___", "**/*___jb_tmp___", "**/.idea/**"} emacsPatterns := []string{"**/.#*"} - vimPatterns := []string{"**/4913", "**/*~", "**/.*.swp", "**/.*.swx"} + // if .swp is taken (presumably because multiple vims are running in that dir), + // vim will go with .swo, .swn, etc, and then even .svz, .svy! + // https://github.com/vim/vim/blob/ea781459b9617aa47335061fcc78403495260315/src/memline.c#L5076 + // ignoring .sw? seems dangerous, since things like .swf or .swi exist, but ignoring the first few + // seems safe and should catch most cases + vimPatterns := []string{"**/4913", "**/*~", "**/.*.swp", "**/.*.swx", "**/.*.swo", "**/.*.swn"} allPatterns := []string{} allPatterns = append(allPatterns, golandPatterns...) From 3f526c5c7bcd88103ebf09f1c207021b728eac9a Mon Sep 17 00:00:00 2001 From: Nick Santos Date: Fri, 15 May 2020 10:34:39 -0400 Subject: [PATCH 05/10] change org name from windmilleng to tilt-dev (#3346) --- pkg/watch/ephemeral.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index 6b260481..a67c4a70 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -1,8 +1,8 @@ package ignore import ( - "github.com/windmilleng/tilt/internal/dockerignore" - "github.com/windmilleng/tilt/pkg/model" + "github.com/tilt-dev/tilt/internal/dockerignore" + "github.com/tilt-dev/tilt/pkg/model" ) // Filter out spurious changes that we don't want to rebuild on, like IDE From 62b5f78fd9b2549e549d80ed8694eef180f6ed7c Mon Sep 17 00:00:00 2001 From: Iggy Jackson Date: Wed, 27 May 2020 07:36:06 -0700 Subject: [PATCH 06/10] Add .kate-swp files to ignore pattern (#3380) KDE's text editor, kate, uses a file similar to Vim's .swp files. Ignore these files so we don't rebuild on every keypress. Fixes #3378 --- pkg/watch/ephemeral.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index a67c4a70..68001101 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -25,11 +25,16 @@ func initEphemeralPathMatcher() model.PathMatcher { // ignoring .sw? seems dangerous, since things like .swf or .swi exist, but ignoring the first few // seems safe and should catch most cases vimPatterns := []string{"**/4913", "**/*~", "**/.*.swp", "**/.*.swx", "**/.*.swo", "**/.*.swn"} + // kate (the default text editor for KDE) uses a file similar to Vim's .swp + // files, but it doesn't have the "incrememnting" character problem mentioned + // above + katePatterns := []string{"**/.*.kate-swp"} allPatterns := []string{} allPatterns = append(allPatterns, golandPatterns...) allPatterns = append(allPatterns, emacsPatterns...) allPatterns = append(allPatterns, vimPatterns...) + allPatterns = append(allPatterns, katePatterns...) matcher, err := dockerignore.NewDockerPatternMatcher("/", allPatterns) if err != nil { From 12de97b8d17d734f23bed9a73b2bf8c1f933ab96 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Tue, 9 Mar 2021 13:05:32 -0500 Subject: [PATCH 07/10] filewatch: use apiserver FileWatch model in EngineState (#4277) This follows the "action-first" approach to use the apiserver model for `FileWatch` and dispatch simple create/update/delete actions. --- pkg/watch/ephemeral.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index 68001101..0d7ce948 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -14,7 +14,7 @@ import ( // there or aren't in the right places. // // https://app.clubhouse.io/windmill/story/691/filter-out-ephemeral-file-changes -var ephemeralPathMatcher = initEphemeralPathMatcher() +var EphemeralPathMatcher = initEphemeralPathMatcher() func initEphemeralPathMatcher() model.PathMatcher { golandPatterns := []string{"**/*___jb_old___", "**/*___jb_tmp___", "**/.idea/**"} From dd5ea044bb5242e87ac1ff8ea2a0256b5def8d21 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Wed, 27 Apr 2022 16:42:51 -0400 Subject: [PATCH 08/10] ignore: add Go umask files to ephemeral set (#5740) When creating files in Go, the stdlib will create (and then rapidly delete) files ending with `-go-tmp-umask` to determine the umask to use for permission purposes. This can cause trouble with Live Update because the files tend to vanish underneath it, for example. Fixes #5117. --- pkg/watch/ephemeral.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index 0d7ce948..637848cd 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -5,8 +5,8 @@ import ( "github.com/tilt-dev/tilt/pkg/model" ) -// Filter out spurious changes that we don't want to rebuild on, like IDE -// temp/lock files. +// EphemeralPathMatcher filters out spurious changes that we don't want to +// rebuild on, like IDE temp/lock files. // // This isn't an ideal solution. In an ideal world, the user would put // everything to ignore in their tiltignore/dockerignore files. This is a @@ -29,12 +29,17 @@ func initEphemeralPathMatcher() model.PathMatcher { // files, but it doesn't have the "incrememnting" character problem mentioned // above katePatterns := []string{"**/.*.kate-swp"} + // go stdlib creates tmpfiles to determine umask for setting permissions + // during file creation; they are then immediately deleted + // https://github.com/golang/go/blob/0b5218cf4e3e5c17344ea113af346e8e0836f6c4/src/cmd/go/internal/work/exec.go#L1764 + goPatterns := []string{"**/*-go-tmp-umask"} allPatterns := []string{} allPatterns = append(allPatterns, golandPatterns...) allPatterns = append(allPatterns, emacsPatterns...) allPatterns = append(allPatterns, vimPatterns...) allPatterns = append(allPatterns, katePatterns...) + allPatterns = append(allPatterns, goPatterns...) matcher, err := dockerignore.NewDockerPatternMatcher("/", allPatterns) if err != nil { From 6fae6a41f959f284e6d49b5dbfb91c32661e42a4 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 26 Jul 2022 08:57:40 -0500 Subject: [PATCH 09/10] Update emacs ignore patterns (#5903) Currently the emacs ignore patterns include `**/.#*` (lock files), but doesn't include `**/#*#` (autosave files; https://www.emacswiki.org/emacs/AutoSave, not to be confused with `**/*~` backup files which are ignored.) Add autosave files. --- pkg/watch/ephemeral.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index 637848cd..cc000a81 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -18,7 +18,7 @@ var EphemeralPathMatcher = initEphemeralPathMatcher() func initEphemeralPathMatcher() model.PathMatcher { golandPatterns := []string{"**/*___jb_old___", "**/*___jb_tmp___", "**/.idea/**"} - emacsPatterns := []string{"**/.#*"} + emacsPatterns := []string{"**/.#*", "**/#*#"} // if .swp is taken (presumably because multiple vims are running in that dir), // vim will go with .swo, .swn, etc, and then even .svz, .svy! // https://github.com/vim/vim/blob/ea781459b9617aa47335061fcc78403495260315/src/memline.c#L5076 From da1ca578b5c687480b4510fd9b81056d8c1fead0 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 24 Feb 2023 11:10:15 -0500 Subject: [PATCH 10/10] watch: ignore ephemeral files & minor output tweaks Big change here is to import the ephemeral ignore set from Tilt. The `.git` directory is also ignored for now: this restriction should probably be lifted and made configurable in the future, but it's not generally important to watch and triggers a LOT of events (e.g. Git creates `index.lock` files that will appear and disappear rapidly as terminals/IDEs/etc interact with Git, even for read-only operations). The Tilt-provided ephemeral file set has been slowly devised over time based on temporary files that can cause trouble. We can also look at a more robust/configurable solution here in the future, but thse provide a reasonable out-of-the-box configuration for the moment. There's also some small tweaks to the output to add missing newlines in a few edge cases and such. Signed-off-by: Milas Bowman --- pkg/compose/watch.go | 32 +++++++++++++++++++++++--------- pkg/watch/ephemeral.go | 27 +++++++++++++++++++-------- pkg/watch/notify.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/watch/watcher_darwin.go | 3 +-- pkg/watch/watcher_naive.go | 2 ++ 5 files changed, 81 insertions(+), 19 deletions(-) diff --git a/pkg/compose/watch.go b/pkg/compose/watch.go index 8b53583d..6b0588c1 100644 --- a/pkg/compose/watch.go +++ b/pkg/compose/watch.go @@ -22,14 +22,15 @@ import ( "time" "github.com/compose-spec/compose-go/types" - "github.com/docker/compose/v2/pkg/api" - "github.com/docker/compose/v2/pkg/utils" - "github.com/docker/compose/v2/pkg/watch" "github.com/jonboulle/clockwork" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" + + "github.com/docker/compose/v2/pkg/api" + "github.com/docker/compose/v2/pkg/utils" + "github.com/docker/compose/v2/pkg/watch" ) type DevelopmentConfig struct { @@ -82,11 +83,24 @@ func (s *composeService) Watch(ctx context.Context, project *types.Project, serv } bc := service.Build.Context - ignore, err := watch.LoadDockerIgnore(bc) + dockerIgnores, err := watch.LoadDockerIgnore(bc) if err != nil { return err } + // add a hardcoded set of ignores on top of what came from .dockerignore + // some of this should likely be configurable (e.g. there could be cases + // where you want `.git` to be synced) but this is suitable for now + dotGitIgnore, err := watch.NewDockerPatternMatcher("/", []string{".git/"}) + if err != nil { + return err + } + ignore := watch.NewCompositeMatcher( + dockerIgnores, + watch.EphemeralPathMatcher, + dotGitIgnore, + ) + watcher, err := watch.NewWatcher([]string{bc}, ignore) if err != nil { return err @@ -109,7 +123,7 @@ func (s *composeService) Watch(ctx context.Context, project *types.Project, serv path := event.Path() for _, trigger := range config.Watch { - logrus.Debugf("change deteced on %s - comparing with %s", path, trigger.Path) + logrus.Debugf("change detected on %s - comparing with %s", path, trigger.Path) if watch.IsChild(trigger.Path, path) { fmt.Fprintf(s.stderr(), "change detected on %s\n", path) @@ -126,7 +140,7 @@ func (s *composeService) Watch(ctx context.Context, project *types.Project, serv Destination: fmt.Sprintf("%s:%s", name, dest), } case WatchActionRebuild: - logrus.Debugf("modified file %s require image to be rebuilt", path) + logrus.Debugf("modified file %s requires image to be rebuilt", path) needRebuild <- name default: return fmt.Errorf("watch action %q is not supported", trigger) @@ -176,7 +190,7 @@ func (s *composeService) makeRebuildFn(ctx context.Context, project *types.Proje Services: services, }) if err != nil { - fmt.Fprintf(s.stderr(), "Build failed") + fmt.Fprintf(s.stderr(), "Build failed\n") } for i, service := range project.Services { if id, ok := imageIds[service.Name]; ok { @@ -196,7 +210,7 @@ func (s *composeService) makeRebuildFn(ctx context.Context, project *types.Proje }, }) if err != nil { - fmt.Fprintf(s.stderr(), "Application failed to start after update") + fmt.Fprintf(s.stderr(), "Application failed to start after update\n") } } } @@ -212,7 +226,7 @@ func (s *composeService) makeSyncFn(ctx context.Context, project *types.Project, if err != nil { return err } - fmt.Fprintf(s.stderr(), "%s updated\n", opt.Source) + fmt.Fprintf(s.stderr(), "%s updated\n", opt.Destination) } } } diff --git a/pkg/watch/ephemeral.go b/pkg/watch/ephemeral.go index cc000a81..2aa27150 100644 --- a/pkg/watch/ephemeral.go +++ b/pkg/watch/ephemeral.go @@ -1,9 +1,20 @@ -package ignore +/* + Copyright 2020 Docker Compose CLI authors -import ( - "github.com/tilt-dev/tilt/internal/dockerignore" - "github.com/tilt-dev/tilt/pkg/model" -) + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package watch // EphemeralPathMatcher filters out spurious changes that we don't want to // rebuild on, like IDE temp/lock files. @@ -16,7 +27,7 @@ import ( // https://app.clubhouse.io/windmill/story/691/filter-out-ephemeral-file-changes var EphemeralPathMatcher = initEphemeralPathMatcher() -func initEphemeralPathMatcher() model.PathMatcher { +func initEphemeralPathMatcher() PathMatcher { golandPatterns := []string{"**/*___jb_old___", "**/*___jb_tmp___", "**/.idea/**"} emacsPatterns := []string{"**/.#*", "**/#*#"} // if .swp is taken (presumably because multiple vims are running in that dir), @@ -34,14 +45,14 @@ func initEphemeralPathMatcher() model.PathMatcher { // https://github.com/golang/go/blob/0b5218cf4e3e5c17344ea113af346e8e0836f6c4/src/cmd/go/internal/work/exec.go#L1764 goPatterns := []string{"**/*-go-tmp-umask"} - allPatterns := []string{} + var allPatterns []string allPatterns = append(allPatterns, golandPatterns...) allPatterns = append(allPatterns, emacsPatterns...) allPatterns = append(allPatterns, vimPatterns...) allPatterns = append(allPatterns, katePatterns...) allPatterns = append(allPatterns, goPatterns...) - matcher, err := dockerignore.NewDockerPatternMatcher("/", allPatterns) + matcher, err := NewDockerPatternMatcher("/", allPatterns) if err != nil { panic(err) } diff --git a/pkg/watch/notify.go b/pkg/watch/notify.go index f09ac98f..3a6398a0 100644 --- a/pkg/watch/notify.go +++ b/pkg/watch/notify.go @@ -106,3 +106,39 @@ func DesiredWindowsBufferSize() int { func IsWindowsShortReadError(err error) bool { return runtime.GOOS == "windows" && !errors.Is(err, fsnotify.ErrEventOverflow) } + +type CompositePathMatcher struct { + Matchers []PathMatcher +} + +func NewCompositeMatcher(matchers ...PathMatcher) PathMatcher { + if len(matchers) == 0 { + return EmptyMatcher{} + } + return CompositePathMatcher{Matchers: matchers} +} + +func (c CompositePathMatcher) Matches(f string) (bool, error) { + for _, t := range c.Matchers { + ret, err := t.Matches(f) + if err != nil { + return false, err + } + if ret { + return true, nil + } + } + return false, nil +} + +func (c CompositePathMatcher) MatchesEntireDir(f string) (bool, error) { + for _, t := range c.Matchers { + matches, err := t.MatchesEntireDir(f) + if matches || err != nil { + return matches, err + } + } + return false, nil +} + +var _ PathMatcher = CompositePathMatcher{} diff --git a/pkg/watch/watcher_darwin.go b/pkg/watch/watcher_darwin.go index 9329a294..55813455 100644 --- a/pkg/watch/watcher_darwin.go +++ b/pkg/watch/watcher_darwin.go @@ -20,7 +20,6 @@ package watch import ( - "fmt" "path/filepath" "time" @@ -53,7 +52,6 @@ func (d *fseventNotify) loop() { } for _, e := range events { - fmt.Println(e) e.Path = filepath.Join("/", e.Path) _, isPathWereWatching := d.pathsWereWatching[e.Path] @@ -67,6 +65,7 @@ func (d *fseventNotify) loop() { if err != nil { logrus.Infof("Error matching path %q: %v", e.Path, err) } else if ignore { + logrus.Tracef("Ignoring event for path: %v", e.Path) continue } diff --git a/pkg/watch/watcher_naive.go b/pkg/watch/watcher_naive.go index 60056df1..e8c7e855 100644 --- a/pkg/watch/watcher_naive.go +++ b/pkg/watch/watcher_naive.go @@ -129,6 +129,7 @@ func (d *naiveNotify) watchRecursively(dir string) error { } if shouldSkipDir { + logrus.Debugf("Ignoring directory and its contents (recursively): %s", path) return filepath.SkipDir } @@ -234,6 +235,7 @@ func (d *naiveNotify) shouldNotify(path string) bool { if err != nil { logrus.Infof("Error matching path %q: %v", path, err) } else if ignore { + logrus.Tracef("Ignoring event for path: %v", path) return false }