From 7d0e1dfc3c2f3e5875850112a93d6d7bd93cc7b0 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 26 Jan 2021 16:46:15 +0100 Subject: [PATCH 1/3] introduce compose logs --tail and --follow options Signed-off-by: Nicolas De Loof --- api/compose/api.go | 2 ++ cli/cmd/compose/compose.go | 2 +- cli/cmd/compose/logs.go | 11 ++++++++++- ecs/aws.go | 2 +- ecs/aws_mock.go | 8 ++++---- ecs/logs.go | 2 +- ecs/sdk.go | 5 ++++- local/compose/logs.go | 3 ++- 8 files changed, 25 insertions(+), 10 deletions(-) diff --git a/api/compose/api.go b/api/compose/api.go index fa70c0a8..acb3d00f 100644 --- a/api/compose/api.go +++ b/api/compose/api.go @@ -122,6 +122,8 @@ type ServiceStatus struct { // LogOptions defines optional parameters for the `Log` API type LogOptions struct { Services []string + Tail string + Follow bool } const ( diff --git a/cli/cmd/compose/compose.go b/cli/cmd/compose/compose.go index bfd0c936..065a0b6d 100644 --- a/cli/cmd/compose/compose.go +++ b/cli/cmd/compose/compose.go @@ -97,7 +97,7 @@ func Command(contextType string) *cobra.Command { stopCommand(&opts), psCommand(&opts), listCommand(), - logsCommand(&opts), + logsCommand(&opts, contextType), convertCommand(&opts), runCommand(&opts), ) diff --git a/cli/cmd/compose/logs.go b/cli/cmd/compose/logs.go index 3da5a3ae..0703ec3c 100644 --- a/cli/cmd/compose/logs.go +++ b/cli/cmd/compose/logs.go @@ -24,15 +24,18 @@ import ( "github.com/docker/compose-cli/api/client" "github.com/docker/compose-cli/api/compose" + "github.com/docker/compose-cli/api/context/store" "github.com/docker/compose-cli/cli/formatter" ) type logsOptions struct { *projectOptions composeOptions + follow bool + tail string } -func logsCommand(p *projectOptions) *cobra.Command { +func logsCommand(p *projectOptions, contextType string) *cobra.Command { opts := logsOptions{ projectOptions: p, } @@ -43,6 +46,10 @@ func logsCommand(p *projectOptions) *cobra.Command { return runLogs(cmd.Context(), opts, args) }, } + logsCmd.Flags().BoolVarP(&opts.follow, "follow", "f", false, "Follow log output.") + if contextType == store.DefaultContextType { + logsCmd.Flags().StringVar(&opts.tail, "tail", "all", "Number of lines to show from the end of the logs for each container.") + } return logsCmd } @@ -59,5 +66,7 @@ func runLogs(ctx context.Context, opts logsOptions, services []string) error { consumer := formatter.NewLogConsumer(ctx, os.Stdout) return c.ComposeService().Logs(ctx, projectName, consumer, compose.LogOptions{ Services: services, + Follow: opts.follow, + Tail: opts.tail, }) } diff --git a/ecs/aws.go b/ecs/aws.go index 505622ba..b9e85990 100644 --- a/ecs/aws.go +++ b/ecs/aws.go @@ -63,7 +63,7 @@ type API interface { InspectSecret(ctx context.Context, id string) (secrets.Secret, error) ListSecrets(ctx context.Context) ([]secrets.Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error - GetLogs(ctx context.Context, name string, consumer func(service, container, message string)) error + GetLogs(ctx context.Context, name string, consumer func(service string, container string, message string), follow bool) error DescribeService(ctx context.Context, cluster string, arn string) (compose.ServiceStatus, error) DescribeServiceTasks(ctx context.Context, cluster string, project string, service string) ([]compose.ContainerSummary, error) getURLWithPortMapping(ctx context.Context, targetGroupArns []string) ([]compose.PortPublisher, error) diff --git a/ecs/aws_mock.go b/ecs/aws_mock.go index 70aa9a83..d857cda1 100644 --- a/ecs/aws_mock.go +++ b/ecs/aws_mock.go @@ -285,17 +285,17 @@ func (mr *MockAPIMockRecorder) GetLoadBalancerURL(arg0, arg1 interface{}) *gomoc } // GetLogs mocks base method -func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 func(string, string, string)) error { +func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 func(string, string, string), arg3 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLogs", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "GetLogs", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // GetLogs indicates an expected call of GetLogs -func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1, arg2, arg3) } // GetParameter mocks base method diff --git a/ecs/logs.go b/ecs/logs.go index 2be5b670..255e66e8 100644 --- a/ecs/logs.go +++ b/ecs/logs.go @@ -26,7 +26,7 @@ func (b *ecsAPIService) Logs(ctx context.Context, projectName string, consumer c if len(options.Services) > 0 { consumer = filteredLogConsumer(consumer, options.Services) } - err := b.aws.GetLogs(ctx, projectName, consumer.Log) + err := b.aws.GetLogs(ctx, projectName, consumer.Log, options.Follow) return err } diff --git a/ecs/sdk.go b/ecs/sdk.go index 0c3ca93e..71d6c62f 100644 --- a/ecs/sdk.go +++ b/ecs/sdk.go @@ -805,7 +805,7 @@ func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { return err } -func (s sdk) GetLogs(ctx context.Context, name string, consumer func(service, container, message string)) error { +func (s sdk) GetLogs(ctx context.Context, name string, consumer func(service string, container string, message string), follow bool) error { logGroup := fmt.Sprintf("/docker-compose/%s", name) var startTime int64 for { @@ -837,6 +837,9 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer func(service, co } } } + if !follow { + return nil + } time.Sleep(500 * time.Millisecond) } } diff --git a/local/compose/logs.go b/local/compose/logs.go index 3b8d34ac..691768c6 100644 --- a/local/compose/logs.go +++ b/local/compose/logs.go @@ -64,7 +64,8 @@ func (s *composeService) Logs(ctx context.Context, projectName string, consumer r, err := s.apiClient.ContainerLogs(ctx, container.ID, types.ContainerLogsOptions{ ShowStdout: true, ShowStderr: true, - Follow: true, + Follow: options.Follow, + Tail: options.Tail, }) defer r.Close() // nolint errcheck From 163f3b9a89bd3890e9ba72a93df966e68f6a446c Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 27 Jan 2021 10:30:39 +0100 Subject: [PATCH 2/3] don't run "removeContainers" in parallel as we follow dependency order Signed-off-by: Nicolas De Loof --- local/compose/create.go | 7 +------ local/compose/down.go | 25 ++++++++++--------------- local/compose/stop.go | 14 +++----------- 3 files changed, 14 insertions(+), 32 deletions(-) diff --git a/local/compose/create.go b/local/compose/create.go index b4b251bd..2e2eea45 100644 --- a/local/compose/create.go +++ b/local/compose/create.go @@ -35,7 +35,6 @@ import ( "github.com/docker/go-connections/nat" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" "github.com/docker/compose-cli/api/compose" "github.com/docker/compose-cli/api/progress" @@ -77,15 +76,11 @@ func (s *composeService) Create(ctx context.Context, project *types.Project, opt orphans := observedState.filter(isNotService(project.ServiceNames()...)) if len(orphans) > 0 { if opts.RemoveOrphans { - eg, _ := errgroup.WithContext(ctx) w := progress.ContextWriter(ctx) - err := s.removeContainers(ctx, w, eg, orphans) + err := s.removeContainers(ctx, w, orphans) if err != nil { return err } - if eg.Wait() != nil { - return err - } } else { logrus.Warnf("Found orphan containers (%s) for this project. If "+ "you removed or renamed this service in your compose "+ diff --git a/local/compose/down.go b/local/compose/down.go index 47581dda..4a544886 100644 --- a/local/compose/down.go +++ b/local/compose/down.go @@ -33,7 +33,6 @@ import ( ) func (s *composeService) Down(ctx context.Context, projectName string, options compose.DownOptions) error { - eg, _ := errgroup.WithContext(ctx) w := progress.ContextWriter(ctx) if options.Project == nil { @@ -55,25 +54,21 @@ func (s *composeService) Down(ctx context.Context, projectName string, options c err = InReverseDependencyOrder(ctx, options.Project, func(c context.Context, service types.ServiceConfig) error { serviceContainers, others := containers.split(isService(service.Name)) - err := s.removeContainers(ctx, w, eg, serviceContainers) + err := s.removeContainers(ctx, w, serviceContainers) containers = others return err }) + if err != nil { + return err + } - if options.RemoveOrphans { - err := s.removeContainers(ctx, w, eg, containers) + if options.RemoveOrphans && len(containers) > 0 { + err := s.removeContainers(ctx, w, containers) if err != nil { return err } } - if err != nil { - return err - } - err = eg.Wait() - if err != nil { - return err - } networks, err := s.apiClient.NetworkList(ctx, moby.NetworkListOptions{ Filters: filters.NewArgs( projectFilter(projectName), @@ -82,6 +77,8 @@ func (s *composeService) Down(ctx context.Context, projectName string, options c if err != nil { return err } + + eg, _ := errgroup.WithContext(ctx) for _, n := range networks { networkID := n.ID networkName := n.Name @@ -89,7 +86,6 @@ func (s *composeService) Down(ctx context.Context, projectName string, options c return s.ensureNetworkDown(ctx, networkID, networkName) }) } - return eg.Wait() } @@ -108,15 +104,14 @@ func (s *composeService) stopContainers(ctx context.Context, w progress.Writer, return nil } -func (s *composeService) removeContainers(ctx context.Context, w progress.Writer, eg *errgroup.Group, containers []moby.Container) error { +func (s *composeService) removeContainers(ctx context.Context, w progress.Writer, containers []moby.Container) error { + eg, _ := errgroup.WithContext(ctx) for _, container := range containers { toDelete := container eg.Go(func() error { eventName := "Container " + getCanonicalContainerName(toDelete) - w.Event(progress.StoppingEvent(eventName)) err := s.stopContainers(ctx, w, []moby.Container{container}) if err != nil { - w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping")) return err } w.Event(progress.RemovingEvent(eventName)) diff --git a/local/compose/stop.go b/local/compose/stop.go index 9ffd6961..e06cd341 100644 --- a/local/compose/stop.go +++ b/local/compose/stop.go @@ -19,17 +19,14 @@ package compose import ( "context" + "github.com/compose-spec/compose-go/types" moby "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/compose-cli/api/progress" - - "github.com/compose-spec/compose-go/types" - "golang.org/x/sync/errgroup" ) -func (s *composeService) Stop(ctx context.Context, project *types.Project) error { - eg, _ := errgroup.WithContext(ctx) +func (s *composeService) Stop(ctx context.Context, project *types.Project, consumer compose.LogConsumer) error { w := progress.ContextWriter(ctx) var containers Containers @@ -41,15 +38,10 @@ func (s *composeService) Stop(ctx context.Context, project *types.Project) error return err } - err = InReverseDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error { + return InReverseDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error { serviceContainers, others := containers.split(isService(service.Name)) err := s.stopContainers(ctx, w, serviceContainers) containers = others return err }) - if err != nil { - return err - } - - return eg.Wait() } From 9d9dbf3a1f7b6658425c1a2ebfac3269a63cd6b8 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 27 Jan 2021 10:46:16 +0100 Subject: [PATCH 3/3] Can't set `-f` as shortcut for `--follow` : conflict with `--file` Signed-off-by: Nicolas De Loof --- cli/cmd/compose/logs.go | 2 +- local/compose/stop.go | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cli/cmd/compose/logs.go b/cli/cmd/compose/logs.go index 0703ec3c..68450435 100644 --- a/cli/cmd/compose/logs.go +++ b/cli/cmd/compose/logs.go @@ -46,7 +46,7 @@ func logsCommand(p *projectOptions, contextType string) *cobra.Command { return runLogs(cmd.Context(), opts, args) }, } - logsCmd.Flags().BoolVarP(&opts.follow, "follow", "f", false, "Follow log output.") + logsCmd.Flags().BoolVar(&opts.follow, "follow", false, "Follow log output.") if contextType == store.DefaultContextType { logsCmd.Flags().StringVar(&opts.tail, "tail", "all", "Number of lines to show from the end of the logs for each container.") } diff --git a/local/compose/stop.go b/local/compose/stop.go index e06cd341..188aabf7 100644 --- a/local/compose/stop.go +++ b/local/compose/stop.go @@ -26,7 +26,7 @@ import ( "github.com/docker/compose-cli/api/progress" ) -func (s *composeService) Stop(ctx context.Context, project *types.Project, consumer compose.LogConsumer) error { +func (s *composeService) Stop(ctx context.Context, project *types.Project) error { w := progress.ContextWriter(ctx) var containers Containers @@ -39,9 +39,6 @@ func (s *composeService) Stop(ctx context.Context, project *types.Project, consu } return InReverseDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error { - serviceContainers, others := containers.split(isService(service.Name)) - err := s.stopContainers(ctx, w, serviceContainers) - containers = others - return err + return s.stopContainers(ctx, w, containers.filter(isService(service.Name))) }) }