From 412385c49549a48e55df45c99c40665a8eb0c36e Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 3 Dec 2020 09:24:15 +0100 Subject: [PATCH 01/19] Add `compose run` command Signed-off-by: aiordache --- aci/compose.go | 7 + api/client/compose.go | 8 ++ api/compose/api.go | 24 ++++ cli/cmd/compose/compose.go | 3 +- cli/cmd/compose/run.go | 136 +++++++++++++++++++ cli/main.go | 3 +- ecs/local/compose.go | 7 + ecs/run.go | 33 +++++ example/backend.go | 7 + local/compose/convergence.go | 4 + local/compose/run.go | 250 +++++++++++++++++++++++++++++++++++ local/compose/util.go | 11 ++ 12 files changed, 491 insertions(+), 2 deletions(-) create mode 100644 cli/cmd/compose/run.go create mode 100644 ecs/run.go create mode 100644 local/compose/run.go diff --git a/aci/compose.go b/aci/compose.go index 92b5d76c..f430ae34 100644 --- a/aci/compose.go +++ b/aci/compose.go @@ -201,3 +201,10 @@ func (cs *aciComposeService) Logs(ctx context.Context, projectName string, consu func (cs *aciComposeService) Convert(ctx context.Context, project *types.Project, options compose.ConvertOptions) ([]byte, error) { return nil, errdefs.ErrNotImplemented } + +func (cs *aciComposeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { + return "", errdefs.ErrNotImplemented +} +func (cs *aciComposeService) Run(ctx context.Context, container string, detach bool) error { + return errdefs.ErrNotImplemented +} diff --git a/api/client/compose.go b/api/client/compose.go index aac8d56e..5693842d 100644 --- a/api/client/compose.go +++ b/api/client/compose.go @@ -71,3 +71,11 @@ func (c *composeService) List(context.Context, string) ([]compose.Stack, error) func (c *composeService) Convert(context.Context, *types.Project, compose.ConvertOptions) ([]byte, error) { return nil, errdefs.ErrNotImplemented } + +func (c *composeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { + return "", errdefs.ErrNotImplemented +} + +func (c *composeService) Run(ctx context.Context, container string, detach bool) error { + return errdefs.ErrNotImplemented +} diff --git a/api/compose/api.go b/api/compose/api.go index 5687cc01..1552cc69 100644 --- a/api/compose/api.go +++ b/api/compose/api.go @@ -18,6 +18,7 @@ package compose import ( "context" + "io" "github.com/compose-spec/compose-go/types" ) @@ -46,6 +47,10 @@ type Service interface { List(ctx context.Context, projectName string) ([]Stack, error) // Convert translate compose model into backend's native format Convert(ctx context.Context, project *types.Project, options ConvertOptions) ([]byte, error) + // CreateOneOffContainer creates a service oneoff container and starts its dependencies + CreateOneOffContainer(ctx context.Context, project *types.Project, opts RunOptions) (string, error) + // Run attaches to and starts a one-off container + Run(ctx context.Context, container string, detach bool) error } // UpOptions group options of the Up API @@ -66,6 +71,25 @@ type ConvertOptions struct { Format string } +// RunOptions holds all flags for compose run +type RunOptions struct { + Name string + Command []string + WorkingDir string + Environment []string + Publish []string + Labels []string + Volumes []string + Remove bool + NoDeps bool + LogConsumer LogConsumer + + Detach bool + + Stdout io.ReadCloser + Stdin io.WriteCloser +} + // PortPublisher hold status about published port type PortPublisher struct { URL string diff --git a/cli/cmd/compose/compose.go b/cli/cmd/compose/compose.go index 2881060c..32b314d4 100644 --- a/cli/cmd/compose/compose.go +++ b/cli/cmd/compose/compose.go @@ -90,6 +90,7 @@ func Command(contextType string) *cobra.Command { listCommand(), logsCommand(), convertCommand(), + runCommand(), ) if contextType == store.LocalContextType || contextType == store.DefaultContextType { @@ -99,7 +100,7 @@ func Command(contextType string) *cobra.Command { pullCommand(), ) } - + command.Flags().SetInterspersed(false) return command } diff --git a/cli/cmd/compose/run.go b/cli/cmd/compose/run.go new file mode 100644 index 00000000..145f9fcd --- /dev/null +++ b/cli/cmd/compose/run.go @@ -0,0 +1,136 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + "fmt" + + "github.com/compose-spec/compose-go/cli" + "github.com/spf13/cobra" + + "github.com/docker/compose-cli/api/client" + "github.com/docker/compose-cli/api/compose" + "github.com/docker/compose-cli/api/containers" + apicontext "github.com/docker/compose-cli/context" + "github.com/docker/compose-cli/context/store" + "github.com/docker/compose-cli/progress" +) + +type runOptions struct { + Name string + Command []string + WorkingDir string + Environment []string + Detach bool + Publish []string + Labels []string + Volumes []string + NoDeps bool + Remove bool +} + +func runCommand() *cobra.Command { + opts := runOptions{} + runCmd := &cobra.Command{ + Use: "run [options] [-v VOLUME...] [-p PORT...] [-e KEY=VAL...] [-l KEY=VALUE...] SERVICE [COMMAND] [ARGS...]", + Short: "Run a one-off command on a service.", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + s := store.ContextStore(cmd.Context()) + currentCtx, err := s.Get(apicontext.CurrentContext(cmd.Context())) + if err != nil { + return err + } + switch currentCtx.Type() { + case store.DefaultContextType: + default: + return fmt.Errorf(`Command "run" is not yet implemented for %q context type`, currentCtx.Type()) + } + + if len(args) > 1 { + opts.Command = args[1:] + } + opts.Name = args[0] + return runRun(cmd.Context(), opts) + }, + } + runCmd.Flags().StringVar(&opts.WorkingDir, "workdir", "", "Work dir") + + runCmd.Flags().StringArrayVarP(&opts.Publish, "publish", "p", []string{}, "Publish a container's port(s). [HOST_PORT:]CONTAINER_PORT") + runCmd.Flags().StringVar(&opts.Name, "name", "", "Assign a name to the container") + runCmd.Flags().BoolVar(&opts.NoDeps, "no-deps", false, "Don't start linked services.") + runCmd.Flags().StringArrayVarP(&opts.Labels, "label", "l", []string{}, "Set meta data on a container") + runCmd.Flags().StringArrayVarP(&opts.Volumes, "volume", "v", []string{}, "Volume. Ex: storageaccount/my_share[:/absolute/path/to/target][:ro]") + runCmd.Flags().BoolVarP(&opts.Detach, "detach", "d", false, "Run container in background and print container ID") + runCmd.Flags().StringArrayVarP(&opts.Environment, "env", "e", []string{}, "Set environment variables") + runCmd.Flags().BoolVar(&opts.Remove, "rm", false, "Automatically remove the container when it exits") + + //addComposeCommonFlags(runCmd.Flags(), &opts.ComposeOpts) + + runCmd.Flags().SetInterspersed(false) + return runCmd +} + +func runRun(ctx context.Context, opts runOptions) error { + // target service + services := []string{opts.Name} + + projectOpts := composeOptions{} + options, err := projectOpts.toProjectOptions() + if err != nil { + return err + } + project, err := cli.ProjectFromOptions(options) + if err != nil { + return err + } + + err = filter(project, services) + if err != nil { + return err + } + + c, err := client.NewWithDefaultLocalBackend(ctx) + if err != nil { + return err + } + containerID, err := progress.Run(ctx, func(ctx context.Context) (string, error) { + return c.ComposeService().CreateOneOffContainer(ctx, project, compose.RunOptions{ + Name: opts.Name, + Command: opts.Command, + }) + }) + if err != nil { + return err + } + // start container and attach to container streams + err = c.ComposeService().Run(ctx, containerID, opts.Detach) + if err != nil { + return err + } + if opts.Detach { + fmt.Printf("%s", containerID) + return nil + } + if opts.Remove { + return c.ContainerService().Delete(ctx, containerID, containers.DeleteRequest{ + Force: true, + }) + } + return nil +} diff --git a/cli/main.go b/cli/main.go index 026af7b9..a0883f22 100644 --- a/cli/main.go +++ b/cli/main.go @@ -150,7 +150,8 @@ func main() { opts.AddContextFlags(root.PersistentFlags()) opts.AddConfigFlags(root.PersistentFlags()) root.Flags().BoolVarP(&opts.Version, "version", "v", false, "Print version information and quit") - + root.PersistentFlags().SetInterspersed(false) + root.Flags().SetInterspersed(false) walk(root, func(c *cobra.Command) { c.Flags().BoolP("help", "h", false, "Help for "+c.Name()) }) diff --git a/ecs/local/compose.go b/ecs/local/compose.go index 1dbaa7fe..9dac1de8 100644 --- a/ecs/local/compose.go +++ b/ecs/local/compose.go @@ -27,6 +27,7 @@ import ( "github.com/compose-spec/compose-go/types" "github.com/docker/compose-cli/api/compose" "github.com/docker/compose-cli/errdefs" + "github.com/pkg/errors" "github.com/sanathkr/go-yaml" ) @@ -162,3 +163,9 @@ func (e ecsLocalSimulation) Ps(ctx context.Context, projectName string) ([]compo func (e ecsLocalSimulation) List(ctx context.Context, projectName string) ([]compose.Stack, error) { return e.compose.List(ctx, projectName) } +func (e ecsLocalSimulation) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { + return "", errors.Wrap(errdefs.ErrNotImplemented, "use docker-compose run") +} +func (e ecsLocalSimulation) Run(ctx context.Context, container string, detach bool) error { + return errdefs.ErrNotImplemented +} diff --git a/ecs/run.go b/ecs/run.go new file mode 100644 index 00000000..643c944c --- /dev/null +++ b/ecs/run.go @@ -0,0 +1,33 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ecs + +import ( + "context" + + "github.com/compose-spec/compose-go/types" + "github.com/docker/compose-cli/api/compose" + "github.com/docker/compose-cli/errdefs" +) + +func (b *ecsAPIService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { + return "", errdefs.ErrNotImplemented +} + +func (b *ecsAPIService) Run(ctx context.Context, container string, detach bool) error { + return errdefs.ErrNotImplemented +} diff --git a/example/backend.go b/example/backend.go index 9e585bd3..55a32d1d 100644 --- a/example/backend.go +++ b/example/backend.go @@ -182,3 +182,10 @@ func (cs *composeService) Logs(ctx context.Context, projectName string, consumer func (cs *composeService) Convert(ctx context.Context, project *types.Project, options compose.ConvertOptions) ([]byte, error) { return nil, errdefs.ErrNotImplemented } +func (cs *composeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { + return "", errdefs.ErrNotImplemented +} + +func (cs *composeService) Run(ctx context.Context, container string, detach bool) error { + return errdefs.ErrNotImplemented +} diff --git a/local/compose/convergence.go b/local/compose/convergence.go index 53df364d..8128d8b7 100644 --- a/local/compose/convergence.go +++ b/local/compose/convergence.go @@ -61,6 +61,10 @@ func (s *composeService) ensureService(ctx context.Context, project *types.Proje for i := 0; i < missing; i++ { number := next + i name := fmt.Sprintf("%s_%s_%d", project.Name, service.Name, number) + if len(service.ContainerName) > 0 { + name = service.ContainerName + } + eg.Go(func() error { return s.createContainer(ctx, project, service, name, number) }) diff --git a/local/compose/run.go b/local/compose/run.go new file mode 100644 index 00000000..4d41949c --- /dev/null +++ b/local/compose/run.go @@ -0,0 +1,250 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + "fmt" + "io" + "os" + "sort" + + "github.com/compose-spec/compose-go/types" + "github.com/docker/compose-cli/api/compose" + convert "github.com/docker/compose-cli/local/moby" + apitypes "github.com/docker/docker/api/types" + moby "github.com/docker/docker/pkg/stringid" +) + +func (s *composeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { + name := opts.Name + service, err := project.GetService(name) + if err != nil { + return "", err + } + + err = s.ensureRequiredNetworks(ctx, project, service) + if err != nil { + return "", err + } + err = s.ensureRequiredVolumes(ctx, project, service) + if err != nil { + return "", err + } + // ensure required services are up and running before creating the oneoff container + err = s.ensureRequiredServices(ctx, project, service) + if err != nil { + return "", err + } + + //apply options to service config + updateOneOffServiceConfig(&service, project.Name, opts) + + err = s.createContainer(ctx, project, service, service.ContainerName, 1) + if err != nil { + return "", err + } + + return service.ContainerName, err +} + +func (s *composeService) Run(ctx context.Context, container string, detach bool) error { + if detach { + // start container + return s.apiClient.ContainerStart(ctx, container, apitypes.ContainerStartOptions{}) + } + + cnx, err := s.apiClient.ContainerAttach(ctx, container, apitypes.ContainerAttachOptions{ + Stream: true, + Stdin: true, + Stdout: true, + Stderr: true, + Logs: true, + }) + if err != nil { + return err + } + defer cnx.Close() + + stdout := convert.ContainerStdout{HijackedResponse: cnx} + stdin := convert.ContainerStdin{HijackedResponse: cnx} + + readChannel := make(chan error, 10) + writeChannel := make(chan error, 10) + + go func() { + _, err := io.Copy(os.Stdout, cnx.Reader) + readChannel <- err + }() + + go func() { + _, err := io.Copy(stdin, os.Stdin) + writeChannel <- err + }() + + go func() { + <-ctx.Done() + stdout.Close() //nolint:errcheck + stdin.Close() //nolint:errcheck + }() + + // start container + err = s.apiClient.ContainerStart(ctx, container, apitypes.ContainerStartOptions{}) + if err != nil { + return err + } + + for { + select { + case err := <-readChannel: + return err + case err := <-writeChannel: + return err + } + } +} + +func updateOneOffServiceConfig(service *types.ServiceConfig, projectName string, opts compose.RunOptions) { + if len(opts.Command) > 0 { + // custom command to run + service.Command = opts.Command + } + //service.Environment = opts.Environment + slug := moby.GenerateRandomID() + service.Scale = 1 + service.ContainerName = fmt.Sprintf("%s_%s_run_%s", projectName, service.Name, moby.TruncateID(slug)) + service.Labels = types.Labels{ + "com.docker.compose.slug": slug, + "com.docker.compose.oneoff": "True", + } + service.Tty = true + service.StdinOpen = true +} + +func (s *composeService) ensureRequiredServices(ctx context.Context, project *types.Project, service types.ServiceConfig) error { + requiredServices := getDependencyNames(project, service, func() []string { + return service.GetDependencies() + }) + if len(requiredServices) > 0 { + // dependencies here + services, err := project.GetServices(requiredServices) + if err != nil { + return err + } + project.Services = services + err = s.ensureImagesExists(ctx, project) + if err != nil { + return err + } + + err = InDependencyOrder(ctx, project, func(c context.Context, svc types.ServiceConfig) error { + return s.ensureService(c, project, svc) + }) + if err != nil { + return err + } + return s.Start(ctx, project, nil) + } + return nil +} + +func (s *composeService) ensureRequiredNetworks(ctx context.Context, project *types.Project, service types.ServiceConfig) error { + networks := getDependentNetworkNames(project, service) + for k, network := range project.Networks { + if !contains(networks, network.Name) { + continue + } + if !network.External.External && network.Name != "" { + network.Name = fmt.Sprintf("%s_%s", project.Name, k) + project.Networks[k] = network + } + network.Labels = network.Labels.Add(networkLabel, k) + network.Labels = network.Labels.Add(projectLabel, project.Name) + network.Labels = network.Labels.Add(versionLabel, ComposeVersion) + + err := s.ensureNetwork(ctx, network) + if err != nil { + return err + } + } + return nil +} + +func (s *composeService) ensureRequiredVolumes(ctx context.Context, project *types.Project, service types.ServiceConfig) error { + volumes := getDependentVolumeNames(project, service) + + for k, volume := range project.Volumes { + if !contains(volumes, volume.Name) { + continue + } + if !volume.External.External && volume.Name != "" { + volume.Name = fmt.Sprintf("%s_%s", project.Name, k) + project.Volumes[k] = volume + } + volume.Labels = volume.Labels.Add(volumeLabel, k) + volume.Labels = volume.Labels.Add(projectLabel, project.Name) + volume.Labels = volume.Labels.Add(versionLabel, ComposeVersion) + err := s.ensureVolume(ctx, volume) + if err != nil { + return err + } + } + return nil +} + +type filterDependency func() []string + +func getDependencyNames(project *types.Project, service types.ServiceConfig, f filterDependency) []string { + names := f() + serviceNames := service.GetDependencies() + if len(serviceNames) == 0 { + return names + } + if len(serviceNames) > 0 { + services, _ := project.GetServices(serviceNames) + for _, s := range services { + svc := getDependencyNames(project, s, f) + names = append(names, svc...) + } + } + sort.Strings(names) + return unique(names) +} + +func getDependentNetworkNames(project *types.Project, service types.ServiceConfig) []string { + return getDependencyNames(project, service, func() []string { + names := []string{} + for n := range service.Networks { + if contains(project.NetworkNames(), n) { + names = append(names, n) + } + } + return names + }) +} + +func getDependentVolumeNames(project *types.Project, service types.ServiceConfig) []string { + return getDependencyNames(project, service, func() []string { + names := []string{} + for _, v := range service.Volumes { + if contains(project.VolumeNames(), v.Source) { + names = append(names, v.Source) + } + } + return names + }) +} diff --git a/local/compose/util.go b/local/compose/util.go index dd9cbbcf..3a025ed6 100644 --- a/local/compose/util.go +++ b/local/compose/util.go @@ -38,3 +38,14 @@ func contains(slice []string, item string) bool { } return false } + +func unique(s []string) []string { + items := []string{} + for _, item := range s { + if contains(items, item) { + continue + } + items = append(items, item) + } + return items +} From 215f50166a130f727b23793035265ea36414c428 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 14:27:21 +0100 Subject: [PATCH 02/19] Remove comment Signed-off-by: Guillaume Tardif --- cli/cmd/compose/run.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/cmd/compose/run.go b/cli/cmd/compose/run.go index 145f9fcd..8623edf7 100644 --- a/cli/cmd/compose/run.go +++ b/cli/cmd/compose/run.go @@ -80,8 +80,6 @@ func runCommand() *cobra.Command { runCmd.Flags().StringArrayVarP(&opts.Environment, "env", "e", []string{}, "Set environment variables") runCmd.Flags().BoolVar(&opts.Remove, "rm", false, "Automatically remove the container when it exits") - //addComposeCommonFlags(runCmd.Flags(), &opts.ComposeOpts) - runCmd.Flags().SetInterspersed(false) return runCmd } From 370781e95e9c06d0bc2c46f5bc3f64c873b24f02 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 17:21:16 +0100 Subject: [PATCH 03/19] =?UTF-8?q?Support=20option=20=E2=80=94workingdir,?= =?UTF-8?q?=20-f,=20no=20need=20to=20check=20backend=20support,=20unimplem?= =?UTF-8?q?ented=20error=20is=20returned=20by=20backbends.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Guillaume Tardif --- cli/cmd/compose/run.go | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/cli/cmd/compose/run.go b/cli/cmd/compose/run.go index 8623edf7..a7bd4fac 100644 --- a/cli/cmd/compose/run.go +++ b/cli/cmd/compose/run.go @@ -26,8 +26,6 @@ import ( "github.com/docker/compose-cli/api/client" "github.com/docker/compose-cli/api/compose" "github.com/docker/compose-cli/api/containers" - apicontext "github.com/docker/compose-cli/context" - "github.com/docker/compose-cli/context/store" "github.com/docker/compose-cli/progress" ) @@ -35,6 +33,7 @@ type runOptions struct { Name string Command []string WorkingDir string + ConfigPaths []string Environment []string Detach bool Publish []string @@ -51,17 +50,6 @@ func runCommand() *cobra.Command { Short: "Run a one-off command on a service.", Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - s := store.ContextStore(cmd.Context()) - currentCtx, err := s.Get(apicontext.CurrentContext(cmd.Context())) - if err != nil { - return err - } - switch currentCtx.Type() { - case store.DefaultContextType: - default: - return fmt.Errorf(`Command "run" is not yet implemented for %q context type`, currentCtx.Type()) - } - if len(args) > 1 { opts.Command = args[1:] } @@ -70,7 +58,7 @@ func runCommand() *cobra.Command { }, } runCmd.Flags().StringVar(&opts.WorkingDir, "workdir", "", "Work dir") - + runCmd.Flags().StringArrayVarP(&opts.ConfigPaths, "file", "f", []string{}, "Compose configuration files") runCmd.Flags().StringArrayVarP(&opts.Publish, "publish", "p", []string{}, "Publish a container's port(s). [HOST_PORT:]CONTAINER_PORT") runCmd.Flags().StringVar(&opts.Name, "name", "", "Assign a name to the container") runCmd.Flags().BoolVar(&opts.NoDeps, "no-deps", false, "Don't start linked services.") @@ -85,10 +73,11 @@ func runCommand() *cobra.Command { } func runRun(ctx context.Context, opts runOptions) error { - // target service - services := []string{opts.Name} - - projectOpts := composeOptions{} + projectOpts := composeOptions{ + ConfigPaths: opts.ConfigPaths, + WorkingDir: opts.WorkingDir, + Environment: opts.Environment, + } options, err := projectOpts.toProjectOptions() if err != nil { return err @@ -98,7 +87,7 @@ func runRun(ctx context.Context, opts runOptions) error { return err } - err = filter(project, services) + err = filter(project, []string{opts.Name}) if err != nil { return err } From b289138ca941d3fecaaf0b419e00716f398bd50f Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 17:22:30 +0100 Subject: [PATCH 04/19] No need to filter services again in backend, filter is done by cli command. Added e2e test, labels one-off and slug Signed-off-by: Guillaume Tardif --- local/compose/create.go | 30 +++- local/compose/labels.go | 1 + local/compose/run.go | 130 +++--------------- local/compose/util.go | 11 -- tests/compose-e2e/compose_test.go | 51 +++++++ .../fixtures/run-test/docker-compose.yml | 24 ++++ 6 files changed, 116 insertions(+), 131 deletions(-) create mode 100644 tests/compose-e2e/fixtures/run-test/docker-compose.yml diff --git a/local/compose/create.go b/local/compose/create.go index 2eafbf2b..26bea4c2 100644 --- a/local/compose/create.go +++ b/local/compose/create.go @@ -44,6 +44,20 @@ func (s *composeService) Create(ctx context.Context, project *types.Project) err return err } + if err := s.ensureProjectNetworks(ctx, project); err != nil { + return err + } + + if err := s.ensureProjectNetworks(ctx, project); err != nil { + return err + } + + return InDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error { + return s.ensureService(c, project, service) + }) +} + +func (s *composeService) ensureProjectNetworks(ctx context.Context, project *types.Project) error { for k, network := range project.Networks { if !network.External.External && network.Name != "" { network.Name = fmt.Sprintf("%s_%s", project.Name, k) @@ -57,7 +71,10 @@ func (s *composeService) Create(ctx context.Context, project *types.Project) err return err } } + return nil +} +func (s *composeService) ensureProjectVolumes(ctx context.Context, project *types.Project) error { for k, volume := range project.Volumes { if !volume.External.External && volume.Name != "" { volume.Name = fmt.Sprintf("%s_%s", project.Name, k) @@ -71,10 +88,7 @@ func (s *composeService) Create(ctx context.Context, project *types.Project) err return err } } - - return InDependencyOrder(ctx, project, func(c context.Context, service types.ServiceConfig) error { - return s.ensureService(c, project, service) - }) + return nil } func getContainerCreateOptions(p *types.Project, s types.ServiceConfig, number int, inherit *moby.Container) (*container.Config, *container.HostConfig, *network.NetworkingConfig, error) { @@ -88,11 +102,15 @@ func getContainerCreateOptions(p *types.Project, s types.ServiceConfig, number i labels[k] = v } - // TODO: change oneoffLabel value for containers started with `docker compose run` labels[projectLabel] = p.Name labels[serviceLabel] = s.Name labels[versionLabel] = ComposeVersion - labels[oneoffLabel] = "False" + if _, ok := s.Labels[oneoffLabel]; ok { + labels[oneoffLabel] = s.Labels[oneoffLabel] + labels[slugLabel] = s.Labels[slugLabel] + } else { + labels[oneoffLabel] = "False" + } labels[configHashLabel] = hash labels[workingDirLabel] = p.WorkingDir labels[configFilesLabel] = strings.Join(p.ComposeFiles, ",") diff --git a/local/compose/labels.go b/local/compose/labels.go index ce36bf49..0928f26f 100644 --- a/local/compose/labels.go +++ b/local/compose/labels.go @@ -25,6 +25,7 @@ import ( const ( containerNumberLabel = "com.docker.compose.container-number" oneoffLabel = "com.docker.compose.oneoff" + slugLabel = "com.docker.compose.slug" projectLabel = "com.docker.compose.project" volumeLabel = "com.docker.compose.volume" workingDirLabel = "com.docker.compose.project.working_dir" diff --git a/local/compose/run.go b/local/compose/run.go index 4d41949c..08584ad9 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -21,7 +21,6 @@ import ( "fmt" "io" "os" - "sort" "github.com/compose-spec/compose-go/types" "github.com/docker/compose-cli/api/compose" @@ -37,12 +36,11 @@ func (s *composeService) CreateOneOffContainer(ctx context.Context, project *typ return "", err } - err = s.ensureRequiredNetworks(ctx, project, service) - if err != nil { + if err := s.ensureProjectNetworks(ctx, project); err != nil { return "", err } - err = s.ensureRequiredVolumes(ctx, project, service) - if err != nil { + + if err := s.ensureProjectVolumes(ctx, project); err != nil { return "", err } // ensure required services are up and running before creating the oneoff container @@ -128,123 +126,27 @@ func updateOneOffServiceConfig(service *types.ServiceConfig, projectName string, service.Scale = 1 service.ContainerName = fmt.Sprintf("%s_%s_run_%s", projectName, service.Name, moby.TruncateID(slug)) service.Labels = types.Labels{ - "com.docker.compose.slug": slug, - "com.docker.compose.oneoff": "True", + slugLabel: slug, + oneoffLabel: "True", } service.Tty = true service.StdinOpen = true } func (s *composeService) ensureRequiredServices(ctx context.Context, project *types.Project, service types.ServiceConfig) error { - requiredServices := getDependencyNames(project, service, func() []string { - return service.GetDependencies() - }) - if len(requiredServices) > 0 { - // dependencies here - services, err := project.GetServices(requiredServices) - if err != nil { - return err - } - project.Services = services - err = s.ensureImagesExists(ctx, project) - if err != nil { - return err - } + err := s.ensureImagesExists(ctx, project) + if err != nil { + return err + } - err = InDependencyOrder(ctx, project, func(c context.Context, svc types.ServiceConfig) error { + err = InDependencyOrder(ctx, project, func(c context.Context, svc types.ServiceConfig) error { + if svc.Name != service.Name { // only start dependencies, not service to run one-off return s.ensureService(c, project, svc) - }) - if err != nil { - return err } - return s.Start(ctx, project, nil) - } - return nil -} - -func (s *composeService) ensureRequiredNetworks(ctx context.Context, project *types.Project, service types.ServiceConfig) error { - networks := getDependentNetworkNames(project, service) - for k, network := range project.Networks { - if !contains(networks, network.Name) { - continue - } - if !network.External.External && network.Name != "" { - network.Name = fmt.Sprintf("%s_%s", project.Name, k) - project.Networks[k] = network - } - network.Labels = network.Labels.Add(networkLabel, k) - network.Labels = network.Labels.Add(projectLabel, project.Name) - network.Labels = network.Labels.Add(versionLabel, ComposeVersion) - - err := s.ensureNetwork(ctx, network) - if err != nil { - return err - } - } - return nil -} - -func (s *composeService) ensureRequiredVolumes(ctx context.Context, project *types.Project, service types.ServiceConfig) error { - volumes := getDependentVolumeNames(project, service) - - for k, volume := range project.Volumes { - if !contains(volumes, volume.Name) { - continue - } - if !volume.External.External && volume.Name != "" { - volume.Name = fmt.Sprintf("%s_%s", project.Name, k) - project.Volumes[k] = volume - } - volume.Labels = volume.Labels.Add(volumeLabel, k) - volume.Labels = volume.Labels.Add(projectLabel, project.Name) - volume.Labels = volume.Labels.Add(versionLabel, ComposeVersion) - err := s.ensureVolume(ctx, volume) - if err != nil { - return err - } - } - return nil -} - -type filterDependency func() []string - -func getDependencyNames(project *types.Project, service types.ServiceConfig, f filterDependency) []string { - names := f() - serviceNames := service.GetDependencies() - if len(serviceNames) == 0 { - return names - } - if len(serviceNames) > 0 { - services, _ := project.GetServices(serviceNames) - for _, s := range services { - svc := getDependencyNames(project, s, f) - names = append(names, svc...) - } - } - sort.Strings(names) - return unique(names) -} - -func getDependentNetworkNames(project *types.Project, service types.ServiceConfig) []string { - return getDependencyNames(project, service, func() []string { - names := []string{} - for n := range service.Networks { - if contains(project.NetworkNames(), n) { - names = append(names, n) - } - } - return names - }) -} - -func getDependentVolumeNames(project *types.Project, service types.ServiceConfig) []string { - return getDependencyNames(project, service, func() []string { - names := []string{} - for _, v := range service.Volumes { - if contains(project.VolumeNames(), v.Source) { - names = append(names, v.Source) - } - } - return names + return nil }) + if err != nil { + return err + } + return s.Start(ctx, project, nil) } diff --git a/local/compose/util.go b/local/compose/util.go index 3a025ed6..dd9cbbcf 100644 --- a/local/compose/util.go +++ b/local/compose/util.go @@ -38,14 +38,3 @@ func contains(slice []string, item string) bool { } return false } - -func unique(s []string) []string { - items := []string{} - for _, item := range s { - if contains(items, item) { - continue - } - items = append(items, item) - } - return items -} diff --git a/tests/compose-e2e/compose_test.go b/tests/compose-e2e/compose_test.go index ddf99b51..c034f02e 100644 --- a/tests/compose-e2e/compose_test.go +++ b/tests/compose-e2e/compose_test.go @@ -103,6 +103,57 @@ func TestLocalComposeUp(t *testing.T) { }) } +func TestLocalComposeRun(t *testing.T) { + c := NewParallelE2eCLI(t, binDir) + + t.Run("compose run", func(t *testing.T) { + res := c.RunDockerCmd("compose", "run", "-f", "./fixtures/run-test/docker-compose.yml", "back") + res.Assert(t, icmd.Expected{Out: "Hello there!!"}) + }) + + t.Run("check run container exited", func(t *testing.T) { + res := c.RunDockerCmd("ps", "--all") + lines := Lines(res.Stdout()) + var runContainerID string + for _, line := range lines { + fields := strings.Fields(line) + containerID := fields[len(fields)-1] + assert.Assert(t, !strings.HasPrefix(containerID, "run-test_front")) + if strings.HasPrefix(containerID, "run-test_back") { + //only the one-off container for back service + assert.Assert(t, strings.HasPrefix(containerID, "run-test_back_run_"), containerID) + runContainerID = containerID + assert.Assert(t, strings.Contains(line, "Exited"), line) + } + if strings.HasPrefix(containerID, "run-test_db_1") { + assert.Assert(t, strings.Contains(line, "Up"), line) + } + } + assert.Assert(t, runContainerID != "") + res = c.RunDockerCmd("inspect", runContainerID) + res.Assert(t, icmd.Expected{Out: `"com.docker.compose.container-number": "1"`}) + res.Assert(t, icmd.Expected{Out: `"com.docker.compose.project": "run-test"`}) + res.Assert(t, icmd.Expected{Out: `"com.docker.compose.oneoff": "True",`}) + res.Assert(t, icmd.Expected{Out: `"com.docker.compose.slug": "`}) + }) + + t.Run("compose run --rm", func(t *testing.T) { + res := c.RunDockerCmd("compose", "run", "-f", "./fixtures/run-test/docker-compose.yml", "--rm", "back") + res.Assert(t, icmd.Expected{Out: "Hello there!!"}) + }) + + t.Run("check run container removed", func(t *testing.T) { + res := c.RunDockerCmd("ps", "--all") + assert.Assert(t, strings.Contains(res.Stdout(), "run-test_back"), res.Stdout()) + }) + + t.Run("down", func(t *testing.T) { + _ = c.RunDockerCmd("compose", "down", "-f", "./fixtures/run-test/docker-compose.yml") + res := c.RunDockerCmd("ps", "--all") + assert.Assert(t, !strings.Contains(res.Stdout(), "run-test"), res.Stdout()) + }) +} + func TestLocalComposeBuild(t *testing.T) { c := NewParallelE2eCLI(t, binDir) diff --git a/tests/compose-e2e/fixtures/run-test/docker-compose.yml b/tests/compose-e2e/fixtures/run-test/docker-compose.yml new file mode 100644 index 00000000..6387a0d7 --- /dev/null +++ b/tests/compose-e2e/fixtures/run-test/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3.8' +services: + back: + image: alpine + command: echo "Hello there!!" + depends_on: + - db + networks: + - backnet + db: + image: nginx + networks: + - backnet + volumes: + - data:/test + front: + image: nginx + networks: + - frontnet +networks: + frontnet: {} + backnet: {} +volumes: + data: {} \ No newline at end of file From 7bed2343e655c9eb97314ca5de9b3fb58a01ba4b Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 17:38:19 +0100 Subject: [PATCH 05/19] Remove not implemented options Signed-off-by: Guillaume Tardif --- api/compose/api.go | 18 ++---------------- cli/cmd/compose/run.go | 9 --------- local/compose/run.go | 1 - 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/api/compose/api.go b/api/compose/api.go index 1552cc69..b0c57d4c 100644 --- a/api/compose/api.go +++ b/api/compose/api.go @@ -18,7 +18,6 @@ package compose import ( "context" - "io" "github.com/compose-spec/compose-go/types" ) @@ -73,21 +72,8 @@ type ConvertOptions struct { // RunOptions holds all flags for compose run type RunOptions struct { - Name string - Command []string - WorkingDir string - Environment []string - Publish []string - Labels []string - Volumes []string - Remove bool - NoDeps bool - LogConsumer LogConsumer - - Detach bool - - Stdout io.ReadCloser - Stdin io.WriteCloser + Name string + Command []string } // PortPublisher hold status about published port diff --git a/cli/cmd/compose/run.go b/cli/cmd/compose/run.go index a7bd4fac..76858a45 100644 --- a/cli/cmd/compose/run.go +++ b/cli/cmd/compose/run.go @@ -36,10 +36,6 @@ type runOptions struct { ConfigPaths []string Environment []string Detach bool - Publish []string - Labels []string - Volumes []string - NoDeps bool Remove bool } @@ -59,11 +55,6 @@ func runCommand() *cobra.Command { } runCmd.Flags().StringVar(&opts.WorkingDir, "workdir", "", "Work dir") runCmd.Flags().StringArrayVarP(&opts.ConfigPaths, "file", "f", []string{}, "Compose configuration files") - runCmd.Flags().StringArrayVarP(&opts.Publish, "publish", "p", []string{}, "Publish a container's port(s). [HOST_PORT:]CONTAINER_PORT") - runCmd.Flags().StringVar(&opts.Name, "name", "", "Assign a name to the container") - runCmd.Flags().BoolVar(&opts.NoDeps, "no-deps", false, "Don't start linked services.") - runCmd.Flags().StringArrayVarP(&opts.Labels, "label", "l", []string{}, "Set meta data on a container") - runCmd.Flags().StringArrayVarP(&opts.Volumes, "volume", "v", []string{}, "Volume. Ex: storageaccount/my_share[:/absolute/path/to/target][:ro]") runCmd.Flags().BoolVarP(&opts.Detach, "detach", "d", false, "Run container in background and print container ID") runCmd.Flags().StringArrayVarP(&opts.Environment, "env", "e", []string{}, "Set environment variables") runCmd.Flags().BoolVar(&opts.Remove, "rm", false, "Automatically remove the container when it exits") diff --git a/local/compose/run.go b/local/compose/run.go index 08584ad9..189a0ecd 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -62,7 +62,6 @@ func (s *composeService) CreateOneOffContainer(ctx context.Context, project *typ func (s *composeService) Run(ctx context.Context, container string, detach bool) error { if detach { - // start container return s.apiClient.ContainerStart(ctx, container, apitypes.ContainerStartOptions{}) } From 53ee1418af1d82c943daa88a7d8edd1d48e366f6 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 17:40:51 +0100 Subject: [PATCH 06/19] Do not change root flags for all commands Signed-off-by: Guillaume Tardif --- cli/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/main.go b/cli/main.go index a0883f22..026af7b9 100644 --- a/cli/main.go +++ b/cli/main.go @@ -150,8 +150,7 @@ func main() { opts.AddContextFlags(root.PersistentFlags()) opts.AddConfigFlags(root.PersistentFlags()) root.Flags().BoolVarP(&opts.Version, "version", "v", false, "Print version information and quit") - root.PersistentFlags().SetInterspersed(false) - root.Flags().SetInterspersed(false) + walk(root, func(c *cobra.Command) { c.Flags().BoolP("help", "h", false, "Help for "+c.Name()) }) From d127cac6d433ab1031169f3ae4467b03bf02e596 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 17:55:55 +0100 Subject: [PATCH 07/19] Ensure images exist before starting progress display for network / volumes, otherwise progress display is all mixed up Signed-off-by: Guillaume Tardif --- local/compose/run.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/local/compose/run.go b/local/compose/run.go index 189a0ecd..065efcb8 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -30,8 +30,11 @@ import ( ) func (s *composeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { - name := opts.Name - service, err := project.GetService(name) + service, err := project.GetService(opts.Name) + if err != nil { + return "", err + } + err = s.ensureImagesExists(ctx, project) if err != nil { return "", err } @@ -133,12 +136,7 @@ func updateOneOffServiceConfig(service *types.ServiceConfig, projectName string, } func (s *composeService) ensureRequiredServices(ctx context.Context, project *types.Project, service types.ServiceConfig) error { - err := s.ensureImagesExists(ctx, project) - if err != nil { - return err - } - - err = InDependencyOrder(ctx, project, func(c context.Context, svc types.ServiceConfig) error { + err := InDependencyOrder(ctx, project, func(c context.Context, svc types.ServiceConfig) error { if svc.Name != service.Name { // only start dependencies, not service to run one-off return s.ensureService(c, project, svc) } From 0e319ae65a018b9441aca536e9e1d9a58b77e1ab Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 17:56:05 +0100 Subject: [PATCH 08/19] Slightly better tests Signed-off-by: Guillaume Tardif --- tests/compose-e2e/compose_test.go | 4 ++-- tests/compose-e2e/fixtures/run-test/docker-compose.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/compose-e2e/compose_test.go b/tests/compose-e2e/compose_test.go index c034f02e..8240fdb3 100644 --- a/tests/compose-e2e/compose_test.go +++ b/tests/compose-e2e/compose_test.go @@ -138,8 +138,8 @@ func TestLocalComposeRun(t *testing.T) { }) t.Run("compose run --rm", func(t *testing.T) { - res := c.RunDockerCmd("compose", "run", "-f", "./fixtures/run-test/docker-compose.yml", "--rm", "back") - res.Assert(t, icmd.Expected{Out: "Hello there!!"}) + res := c.RunDockerCmd("compose", "run", "-f", "./fixtures/run-test/docker-compose.yml", "--rm", "back", "echo", "Hello again") + res.Assert(t, icmd.Expected{Out: "Hello again"}) }) t.Run("check run container removed", func(t *testing.T) { diff --git a/tests/compose-e2e/fixtures/run-test/docker-compose.yml b/tests/compose-e2e/fixtures/run-test/docker-compose.yml index 6387a0d7..7b6e3ac6 100644 --- a/tests/compose-e2e/fixtures/run-test/docker-compose.yml +++ b/tests/compose-e2e/fixtures/run-test/docker-compose.yml @@ -18,7 +18,7 @@ services: networks: - frontnet networks: - frontnet: {} - backnet: {} + frontnet: + backnet: volumes: - data: {} \ No newline at end of file + data: From 8259604074102a5900fd8cb0b32cd7d35aa180e6 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 18:02:56 +0100 Subject: [PATCH 09/19] Simplify command setup Signed-off-by: Guillaume Tardif --- cli/cmd/compose/run.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/cli/cmd/compose/run.go b/cli/cmd/compose/run.go index 76858a45..3979ff75 100644 --- a/cli/cmd/compose/run.go +++ b/cli/cmd/compose/run.go @@ -20,10 +20,8 @@ import ( "context" "fmt" - "github.com/compose-spec/compose-go/cli" "github.com/spf13/cobra" - "github.com/docker/compose-cli/api/client" "github.com/docker/compose-cli/api/compose" "github.com/docker/compose-cli/api/containers" "github.com/docker/compose-cli/progress" @@ -69,24 +67,11 @@ func runRun(ctx context.Context, opts runOptions) error { WorkingDir: opts.WorkingDir, Environment: opts.Environment, } - options, err := projectOpts.toProjectOptions() - if err != nil { - return err - } - project, err := cli.ProjectFromOptions(options) + c, project, err := setup(ctx, projectOpts, []string{opts.Name}) if err != nil { return err } - err = filter(project, []string{opts.Name}) - if err != nil { - return err - } - - c, err := client.NewWithDefaultLocalBackend(ctx) - if err != nil { - return err - } containerID, err := progress.Run(ctx, func(ctx context.Context) (string, error) { return c.ComposeService().CreateOneOffContainer(ctx, project, compose.RunOptions{ Name: opts.Name, From 8eb973391407ec5e0e76c55e6aa816ee4cffc3ff Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 18:03:25 +0100 Subject: [PATCH 10/19] No need to support service `container_name` in this PR, will do separately, minor cleanup Signed-off-by: Guillaume Tardif --- local/compose/convergence.go | 4 ---- local/compose/create.go | 5 +---- local/compose/run.go | 12 ++---------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/local/compose/convergence.go b/local/compose/convergence.go index 8128d8b7..53df364d 100644 --- a/local/compose/convergence.go +++ b/local/compose/convergence.go @@ -61,10 +61,6 @@ func (s *composeService) ensureService(ctx context.Context, project *types.Proje for i := 0; i < missing; i++ { number := next + i name := fmt.Sprintf("%s_%s_%d", project.Name, service.Name, number) - if len(service.ContainerName) > 0 { - name = service.ContainerName - } - eg.Go(func() error { return s.createContainer(ctx, project, service, name, number) }) diff --git a/local/compose/create.go b/local/compose/create.go index 26bea4c2..18fa1b53 100644 --- a/local/compose/create.go +++ b/local/compose/create.go @@ -105,10 +105,7 @@ func getContainerCreateOptions(p *types.Project, s types.ServiceConfig, number i labels[projectLabel] = p.Name labels[serviceLabel] = s.Name labels[versionLabel] = ComposeVersion - if _, ok := s.Labels[oneoffLabel]; ok { - labels[oneoffLabel] = s.Labels[oneoffLabel] - labels[slugLabel] = s.Labels[slugLabel] - } else { + if _, ok := s.Labels[oneoffLabel]; !ok { labels[oneoffLabel] = "False" } labels[configHashLabel] = hash diff --git a/local/compose/run.go b/local/compose/run.go index 065efcb8..a1b5ef6b 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -34,25 +34,19 @@ func (s *composeService) CreateOneOffContainer(ctx context.Context, project *typ if err != nil { return "", err } - err = s.ensureImagesExists(ctx, project) - if err != nil { + if err = s.ensureImagesExists(ctx, project); err != nil { return "", err } - if err := s.ensureProjectNetworks(ctx, project); err != nil { return "", err } - if err := s.ensureProjectVolumes(ctx, project); err != nil { return "", err } - // ensure required services are up and running before creating the oneoff container - err = s.ensureRequiredServices(ctx, project, service) - if err != nil { + if err = s.ensureRequiredServices(ctx, project, service); err != nil { return "", err } - //apply options to service config updateOneOffServiceConfig(&service, project.Name, opts) err = s.createContainer(ctx, project, service, service.ContainerName, 1) @@ -120,10 +114,8 @@ func (s *composeService) Run(ctx context.Context, container string, detach bool) func updateOneOffServiceConfig(service *types.ServiceConfig, projectName string, opts compose.RunOptions) { if len(opts.Command) > 0 { - // custom command to run service.Command = opts.Command } - //service.Environment = opts.Environment slug := moby.GenerateRandomID() service.Scale = 1 service.ContainerName = fmt.Sprintf("%s_%s_run_%s", projectName, service.Name, moby.TruncateID(slug)) From 1e6c4263cba348a1a5498fd73058e4e609ee715c Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Mon, 14 Dec 2020 18:31:35 +0100 Subject: [PATCH 11/19] =?UTF-8?q?Also=20test=20running=20bash=20commands?= =?UTF-8?q?=20with=20`run=20/bin/sh=20-c=20=E2=80=A6`,=20better=20test=20o?= =?UTF-8?q?n=20slug=20label?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Guillaume Tardif --- tests/compose-e2e/compose_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/compose-e2e/compose_test.go b/tests/compose-e2e/compose_test.go index 8240fdb3..8ef6e9d4 100644 --- a/tests/compose-e2e/compose_test.go +++ b/tests/compose-e2e/compose_test.go @@ -115,6 +115,7 @@ func TestLocalComposeRun(t *testing.T) { res := c.RunDockerCmd("ps", "--all") lines := Lines(res.Stdout()) var runContainerID string + var truncatedSlug string for _, line := range lines { fields := strings.Fields(line) containerID := fields[len(fields)-1] @@ -122,6 +123,7 @@ func TestLocalComposeRun(t *testing.T) { if strings.HasPrefix(containerID, "run-test_back") { //only the one-off container for back service assert.Assert(t, strings.HasPrefix(containerID, "run-test_back_run_"), containerID) + truncatedSlug = strings.Replace(containerID, "run-test_back_run_", "", 1) runContainerID = containerID assert.Assert(t, strings.Contains(line, "Exited"), line) } @@ -134,12 +136,13 @@ func TestLocalComposeRun(t *testing.T) { res.Assert(t, icmd.Expected{Out: `"com.docker.compose.container-number": "1"`}) res.Assert(t, icmd.Expected{Out: `"com.docker.compose.project": "run-test"`}) res.Assert(t, icmd.Expected{Out: `"com.docker.compose.oneoff": "True",`}) - res.Assert(t, icmd.Expected{Out: `"com.docker.compose.slug": "`}) + res.Assert(t, icmd.Expected{Out: `"com.docker.compose.slug": "` + truncatedSlug}) }) t.Run("compose run --rm", func(t *testing.T) { - res := c.RunDockerCmd("compose", "run", "-f", "./fixtures/run-test/docker-compose.yml", "--rm", "back", "echo", "Hello again") - res.Assert(t, icmd.Expected{Out: "Hello again"}) + res := c.RunDockerCmd("compose", "run", "-f", "./fixtures/run-test/docker-compose.yml", "--rm", "back", "/bin/sh", "-c", "echo Hello again") + lines := Lines(res.Stdout()) + assert.Equal(t, lines[len(lines)-1], "Hello again") }) t.Run("check run container removed", func(t *testing.T) { From 27dc2e5be1635ea4bedddc3a3d1abc435b365b9a Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Wed, 16 Dec 2020 11:57:58 +0100 Subject: [PATCH 12/19] Reuse Create(Project) and Start(Project) Signed-off-by: Guillaume Tardif --- local/compose/create.go | 2 +- local/compose/run.go | 48 +++++++++++++++++------------------------ 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/local/compose/create.go b/local/compose/create.go index 18fa1b53..6cccef1c 100644 --- a/local/compose/create.go +++ b/local/compose/create.go @@ -48,7 +48,7 @@ func (s *composeService) Create(ctx context.Context, project *types.Project) err return err } - if err := s.ensureProjectNetworks(ctx, project); err != nil { + if err := s.ensureProjectVolumes(ctx, project); err != nil { return err } diff --git a/local/compose/run.go b/local/compose/run.go index a1b5ef6b..77682ff1 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -30,31 +30,36 @@ import ( ) func (s *composeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { - service, err := project.GetService(opts.Name) - if err != nil { + originalServices := project.Services + dependencies := []types.ServiceConfig{} + var requestedService types.ServiceConfig + for _, service := range originalServices { + if service.Name != opts.Name { + dependencies = append(dependencies, service) + } else { + requestedService = service + } + } + project.Services = types.Services(dependencies) + if err := s.Create(ctx, project); err != nil { return "", err } - if err = s.ensureImagesExists(ctx, project); err != nil { - return "", err - } - if err := s.ensureProjectNetworks(ctx, project); err != nil { - return "", err - } - if err := s.ensureProjectVolumes(ctx, project); err != nil { - return "", err - } - if err = s.ensureRequiredServices(ctx, project, service); err != nil { + if err := s.Start(ctx, project, nil); err != nil { return "", err } - updateOneOffServiceConfig(&service, project.Name, opts) + project.Services = originalServices + updateOneOffServiceConfig(&requestedService, project.Name, opts) - err = s.createContainer(ctx, project, service, service.ContainerName, 1) + if err := s.waitDependencies(ctx, project, requestedService); err != nil { + return "", err + } + err := s.createContainer(ctx, project, requestedService, requestedService.ContainerName, 1) if err != nil { return "", err } - return service.ContainerName, err + return requestedService.ContainerName, err } func (s *composeService) Run(ctx context.Context, container string, detach bool) error { @@ -126,16 +131,3 @@ func updateOneOffServiceConfig(service *types.ServiceConfig, projectName string, service.Tty = true service.StdinOpen = true } - -func (s *composeService) ensureRequiredServices(ctx context.Context, project *types.Project, service types.ServiceConfig) error { - err := InDependencyOrder(ctx, project, func(c context.Context, svc types.ServiceConfig) error { - if svc.Name != service.Name { // only start dependencies, not service to run one-off - return s.ensureService(c, project, svc) - } - return nil - }) - if err != nil { - return err - } - return s.Start(ctx, project, nil) -} From 76f36a69c6f92ce3a082daf88d0bfeae47b0ff9a Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Wed, 16 Dec 2020 15:25:31 +0100 Subject: [PATCH 13/19] Reusing existing Create() and Start() API Signed-off-by: Guillaume Tardif --- aci/compose.go | 5 +--- api/client/compose.go | 6 +---- api/compose/api.go | 7 +++-- cli/cmd/compose/run.go | 24 +++++++++++++---- ecs/local/compose.go | 5 +--- ecs/run.go | 6 +---- example/backend.go | 6 +---- local/compose/run.go | 61 +++++++++++++++--------------------------- 8 files changed, 49 insertions(+), 71 deletions(-) diff --git a/aci/compose.go b/aci/compose.go index f430ae34..a8e65c2a 100644 --- a/aci/compose.go +++ b/aci/compose.go @@ -202,9 +202,6 @@ func (cs *aciComposeService) Convert(ctx context.Context, project *types.Project return nil, errdefs.ErrNotImplemented } -func (cs *aciComposeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { +func (cs *aciComposeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { return "", errdefs.ErrNotImplemented } -func (cs *aciComposeService) Run(ctx context.Context, container string, detach bool) error { - return errdefs.ErrNotImplemented -} diff --git a/api/client/compose.go b/api/client/compose.go index 5693842d..2525b0d5 100644 --- a/api/client/compose.go +++ b/api/client/compose.go @@ -72,10 +72,6 @@ func (c *composeService) Convert(context.Context, *types.Project, compose.Conver return nil, errdefs.ErrNotImplemented } -func (c *composeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { +func (c *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { return "", errdefs.ErrNotImplemented } - -func (c *composeService) Run(ctx context.Context, container string, detach bool) error { - return errdefs.ErrNotImplemented -} diff --git a/api/compose/api.go b/api/compose/api.go index b0c57d4c..f1b8e667 100644 --- a/api/compose/api.go +++ b/api/compose/api.go @@ -46,10 +46,8 @@ type Service interface { List(ctx context.Context, projectName string) ([]Stack, error) // Convert translate compose model into backend's native format Convert(ctx context.Context, project *types.Project, options ConvertOptions) ([]byte, error) - // CreateOneOffContainer creates a service oneoff container and starts its dependencies - CreateOneOffContainer(ctx context.Context, project *types.Project, opts RunOptions) (string, error) - // Run attaches to and starts a one-off container - Run(ctx context.Context, container string, detach bool) error + // RunOneOffContainer creates a service oneoff container and starts its dependencies + RunOneOffContainer(ctx context.Context, project *types.Project, opts RunOptions) (string, error) } // UpOptions group options of the Up API @@ -74,6 +72,7 @@ type ConvertOptions struct { type RunOptions struct { Name string Command []string + Detach bool } // PortPublisher hold status about published port diff --git a/cli/cmd/compose/run.go b/cli/cmd/compose/run.go index 3979ff75..42e47842 100644 --- a/cli/cmd/compose/run.go +++ b/cli/cmd/compose/run.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "github.com/compose-spec/compose-go/types" "github.com/spf13/cobra" "github.com/docker/compose-cli/api/compose" @@ -72,17 +73,30 @@ func runRun(ctx context.Context, opts runOptions) error { return err } + dependencies := []types.ServiceConfig{} + originalServices := project.Services containerID, err := progress.Run(ctx, func(ctx context.Context) (string, error) { - return c.ComposeService().CreateOneOffContainer(ctx, project, compose.RunOptions{ - Name: opts.Name, - Command: opts.Command, - }) + for _, service := range originalServices { + if service.Name != opts.Name { + dependencies = append(dependencies, service) + } + } + project.Services = types.Services(dependencies) + if err := c.ComposeService().Create(ctx, project); err != nil { + return "", err + } + if err := c.ComposeService().Start(ctx, project, nil); err != nil { + return "", err + } + return "", nil }) if err != nil { return err } + + project.Services = originalServices // start container and attach to container streams - err = c.ComposeService().Run(ctx, containerID, opts.Detach) + containerID, err = c.ComposeService().RunOneOffContainer(ctx, project, compose.RunOptions{Name: opts.Name, Command: opts.Command, Detach: opts.Detach}) if err != nil { return err } diff --git a/ecs/local/compose.go b/ecs/local/compose.go index 9dac1de8..8f5c5bd3 100644 --- a/ecs/local/compose.go +++ b/ecs/local/compose.go @@ -163,9 +163,6 @@ func (e ecsLocalSimulation) Ps(ctx context.Context, projectName string) ([]compo func (e ecsLocalSimulation) List(ctx context.Context, projectName string) ([]compose.Stack, error) { return e.compose.List(ctx, projectName) } -func (e ecsLocalSimulation) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { +func (e ecsLocalSimulation) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { return "", errors.Wrap(errdefs.ErrNotImplemented, "use docker-compose run") } -func (e ecsLocalSimulation) Run(ctx context.Context, container string, detach bool) error { - return errdefs.ErrNotImplemented -} diff --git a/ecs/run.go b/ecs/run.go index 643c944c..4ce83c33 100644 --- a/ecs/run.go +++ b/ecs/run.go @@ -24,10 +24,6 @@ import ( "github.com/docker/compose-cli/errdefs" ) -func (b *ecsAPIService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { +func (b *ecsAPIService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { return "", errdefs.ErrNotImplemented } - -func (b *ecsAPIService) Run(ctx context.Context, container string, detach bool) error { - return errdefs.ErrNotImplemented -} diff --git a/example/backend.go b/example/backend.go index 55a32d1d..2aba4a76 100644 --- a/example/backend.go +++ b/example/backend.go @@ -182,10 +182,6 @@ func (cs *composeService) Logs(ctx context.Context, projectName string, consumer func (cs *composeService) Convert(ctx context.Context, project *types.Project, options compose.ConvertOptions) ([]byte, error) { return nil, errdefs.ErrNotImplemented } -func (cs *composeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { +func (cs *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { return "", errdefs.ErrNotImplemented } - -func (cs *composeService) Run(ctx context.Context, container string, detach bool) error { - return errdefs.ErrNotImplemented -} diff --git a/local/compose/run.go b/local/compose/run.go index 77682ff1..2c7b1e6d 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -29,27 +29,27 @@ import ( moby "github.com/docker/docker/pkg/stringid" ) -func (s *composeService) CreateOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { +func (s *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { originalServices := project.Services - dependencies := []types.ServiceConfig{} var requestedService types.ServiceConfig for _, service := range originalServices { - if service.Name != opts.Name { - dependencies = append(dependencies, service) - } else { + if service.Name == opts.Name { requestedService = service } } - project.Services = types.Services(dependencies) - if err := s.Create(ctx, project); err != nil { - return "", err - } - if err := s.Start(ctx, project, nil); err != nil { - return "", err - } project.Services = originalServices - updateOneOffServiceConfig(&requestedService, project.Name, opts) + if len(opts.Command) > 0 { + requestedService.Command = opts.Command + } + requestedService.Scale = 1 + requestedService.Tty = true + requestedService.StdinOpen = true + + slug := moby.GenerateRandomID() + requestedService.ContainerName = fmt.Sprintf("%s_%s_run_%s", project.Name, requestedService.Name, moby.TruncateID(slug)) + requestedService.Labels = requestedService.Labels.Add(slugLabel, slug) + requestedService.Labels = requestedService.Labels.Add(oneoffLabel, "True") if err := s.waitDependencies(ctx, project, requestedService); err != nil { return "", err @@ -59,15 +59,13 @@ func (s *composeService) CreateOneOffContainer(ctx context.Context, project *typ return "", err } - return requestedService.ContainerName, err -} + containerID := requestedService.ContainerName -func (s *composeService) Run(ctx context.Context, container string, detach bool) error { - if detach { - return s.apiClient.ContainerStart(ctx, container, apitypes.ContainerStartOptions{}) + if opts.Detach { + return containerID, s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}) } - cnx, err := s.apiClient.ContainerAttach(ctx, container, apitypes.ContainerAttachOptions{ + cnx, err := s.apiClient.ContainerAttach(ctx, containerID, apitypes.ContainerAttachOptions{ Stream: true, Stdin: true, Stdout: true, @@ -75,7 +73,7 @@ func (s *composeService) Run(ctx context.Context, container string, detach bool) Logs: true, }) if err != nil { - return err + return containerID, err } defer cnx.Close() @@ -102,32 +100,17 @@ func (s *composeService) Run(ctx context.Context, container string, detach bool) }() // start container - err = s.apiClient.ContainerStart(ctx, container, apitypes.ContainerStartOptions{}) + err = s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}) if err != nil { - return err + return containerID, err } for { select { case err := <-readChannel: - return err + return containerID, err case err := <-writeChannel: - return err + return containerID, err } } } - -func updateOneOffServiceConfig(service *types.ServiceConfig, projectName string, opts compose.RunOptions) { - if len(opts.Command) > 0 { - service.Command = opts.Command - } - slug := moby.GenerateRandomID() - service.Scale = 1 - service.ContainerName = fmt.Sprintf("%s_%s_run_%s", projectName, service.Name, moby.TruncateID(slug)) - service.Labels = types.Labels{ - slugLabel: slug, - oneoffLabel: "True", - } - service.Tty = true - service.StdinOpen = true -} From d2cfffafb4130d543c6a7ca51c98a8c39d0bb650 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Wed, 16 Dec 2020 17:40:49 +0100 Subject: [PATCH 14/19] Attach to container using compose attach Signed-off-by: Guillaume Tardif --- local/compose/run.go | 73 +++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/local/compose/run.go b/local/compose/run.go index 2c7b1e6d..80417469 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -19,13 +19,15 @@ package compose import ( "context" "fmt" - "io" "os" "github.com/compose-spec/compose-go/types" "github.com/docker/compose-cli/api/compose" - convert "github.com/docker/compose-cli/local/moby" + "github.com/docker/compose-cli/utils" apitypes "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "golang.org/x/sync/errgroup" + moby "github.com/docker/docker/pkg/stringid" ) @@ -65,52 +67,33 @@ func (s *composeService) RunOneOffContainer(ctx context.Context, project *types. return containerID, s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}) } - cnx, err := s.apiClient.ContainerAttach(ctx, containerID, apitypes.ContainerAttachOptions{ - Stream: true, - Stdin: true, - Stdout: true, - Stderr: true, - Logs: true, + containers, err := s.apiClient.ContainerList(ctx, apitypes.ContainerListOptions{ + Filters: filters.NewArgs( + projectFilter(project.Name), + ), + All: true, }) if err != nil { - return containerID, err + return "", err } - defer cnx.Close() - - stdout := convert.ContainerStdout{HijackedResponse: cnx} - stdin := convert.ContainerStdin{HijackedResponse: cnx} - - readChannel := make(chan error, 10) - writeChannel := make(chan error, 10) - - go func() { - _, err := io.Copy(os.Stdout, cnx.Reader) - readChannel <- err - }() - - go func() { - _, err := io.Copy(stdin, os.Stdin) - writeChannel <- err - }() - - go func() { - <-ctx.Done() - stdout.Close() //nolint:errcheck - stdin.Close() //nolint:errcheck - }() - - // start container - err = s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}) - if err != nil { - return containerID, err - } - - for { - select { - case err := <-readChannel: - return containerID, err - case err := <-writeChannel: - return containerID, err + var oneoffContainer apitypes.Container + for _, container := range containers { + if utils.StringContains(container.Names, "/"+containerID) { + oneoffContainer = container } } + eg := errgroup.Group{} + eg.Go(func() error { + return s.attachContainerStreams(ctx, oneoffContainer, true, os.Stdin, os.Stdout) + }) + if err != nil { + return "", err + } + + err = s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}) + if err != nil { + return "", err + } + err = eg.Wait() + return containerID, err } From bad0d41d904d1f28598f589898089727c7295c23 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Wed, 16 Dec 2020 18:34:14 +0100 Subject: [PATCH 15/19] Auto-remove using container config Signed-off-by: Guillaume Tardif --- api/compose/api.go | 7 ++++--- cli/cmd/compose/run.go | 16 +++++----------- local/compose/convergence.go | 12 ++++++------ local/compose/create.go | 3 ++- local/compose/run.go | 16 ++++------------ 5 files changed, 21 insertions(+), 33 deletions(-) diff --git a/api/compose/api.go b/api/compose/api.go index f1b8e667..daea325d 100644 --- a/api/compose/api.go +++ b/api/compose/api.go @@ -70,9 +70,10 @@ type ConvertOptions struct { // RunOptions holds all flags for compose run type RunOptions struct { - Name string - Command []string - Detach bool + Name string + Command []string + Detach bool + AutoRemove bool } // PortPublisher hold status about published port diff --git a/cli/cmd/compose/run.go b/cli/cmd/compose/run.go index 42e47842..437315fb 100644 --- a/cli/cmd/compose/run.go +++ b/cli/cmd/compose/run.go @@ -24,7 +24,6 @@ import ( "github.com/spf13/cobra" "github.com/docker/compose-cli/api/compose" - "github.com/docker/compose-cli/api/containers" "github.com/docker/compose-cli/progress" ) @@ -73,15 +72,15 @@ func runRun(ctx context.Context, opts runOptions) error { return err } - dependencies := []types.ServiceConfig{} originalServices := project.Services - containerID, err := progress.Run(ctx, func(ctx context.Context) (string, error) { + _, err = progress.Run(ctx, func(ctx context.Context) (string, error) { + dependencies := types.Services{} for _, service := range originalServices { if service.Name != opts.Name { dependencies = append(dependencies, service) } } - project.Services = types.Services(dependencies) + project.Services = dependencies if err := c.ComposeService().Create(ctx, project); err != nil { return "", err } @@ -96,18 +95,13 @@ func runRun(ctx context.Context, opts runOptions) error { project.Services = originalServices // start container and attach to container streams - containerID, err = c.ComposeService().RunOneOffContainer(ctx, project, compose.RunOptions{Name: opts.Name, Command: opts.Command, Detach: opts.Detach}) + runOpts := compose.RunOptions{Name: opts.Name, Command: opts.Command, Detach: opts.Detach, AutoRemove: opts.Remove} + containerID, err := c.ComposeService().RunOneOffContainer(ctx, project, runOpts) if err != nil { return err } if opts.Detach { fmt.Printf("%s", containerID) - return nil - } - if opts.Remove { - return c.ContainerService().Delete(ctx, containerID, containers.DeleteRequest{ - Force: true, - }) } return nil } diff --git a/local/compose/convergence.go b/local/compose/convergence.go index 53df364d..826251c6 100644 --- a/local/compose/convergence.go +++ b/local/compose/convergence.go @@ -62,7 +62,7 @@ func (s *composeService) ensureService(ctx context.Context, project *types.Proje number := next + i name := fmt.Sprintf("%s_%s_%d", project.Name, service.Name, number) eg.Go(func() error { - return s.createContainer(ctx, project, service, name, number) + return s.createContainer(ctx, project, service, name, number, false) }) } } @@ -163,10 +163,10 @@ func getScale(config types.ServiceConfig) int { return 1 } -func (s *composeService) createContainer(ctx context.Context, project *types.Project, service types.ServiceConfig, name string, number int) error { +func (s *composeService) createContainer(ctx context.Context, project *types.Project, service types.ServiceConfig, name string, number int, autoRemove bool) error { w := progress.ContextWriter(ctx) w.Event(progress.CreatingEvent(name)) - err := s.runContainer(ctx, project, service, name, number, nil) + err := s.createMobyContainer(ctx, project, service, name, number, nil, autoRemove) if err != nil { return err } @@ -191,7 +191,7 @@ func (s *composeService) recreateContainer(ctx context.Context, project *types.P if err != nil { return err } - err = s.runContainer(ctx, project, service, name, number, &container) + err = s.createMobyContainer(ctx, project, service, name, number, &container, false) if err != nil { return err } @@ -228,8 +228,8 @@ func (s *composeService) restartContainer(ctx context.Context, container moby.Co return nil } -func (s *composeService) runContainer(ctx context.Context, project *types.Project, service types.ServiceConfig, name string, number int, container *moby.Container) error { - containerConfig, hostConfig, networkingConfig, err := getContainerCreateOptions(project, service, number, container) +func (s *composeService) createMobyContainer(ctx context.Context, project *types.Project, service types.ServiceConfig, name string, number int, container *moby.Container, autoRemove bool) error { + containerConfig, hostConfig, networkingConfig, err := getContainerCreateOptions(project, service, number, container, autoRemove) if err != nil { return err } diff --git a/local/compose/create.go b/local/compose/create.go index 6cccef1c..25897c49 100644 --- a/local/compose/create.go +++ b/local/compose/create.go @@ -91,7 +91,7 @@ func (s *composeService) ensureProjectVolumes(ctx context.Context, project *type return nil } -func getContainerCreateOptions(p *types.Project, s types.ServiceConfig, number int, inherit *moby.Container) (*container.Config, *container.HostConfig, *network.NetworkingConfig, error) { +func getContainerCreateOptions(p *types.Project, s types.ServiceConfig, number int, inherit *moby.Container, autoRemove bool) (*container.Config, *container.HostConfig, *network.NetworkingConfig, error) { hash, err := jsonHash(s) if err != nil { return nil, nil, nil, err @@ -167,6 +167,7 @@ func getContainerCreateOptions(p *types.Project, s types.ServiceConfig, number i networkMode := getNetworkMode(p, s) hostConfig := container.HostConfig{ + AutoRemove: autoRemove, Mounts: mountOptions, CapAdd: strslice.StrSlice(s.CapAdd), CapDrop: strslice.StrSlice(s.CapDrop), diff --git a/local/compose/run.go b/local/compose/run.go index 80417469..38260e1a 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -23,7 +23,6 @@ import ( "github.com/compose-spec/compose-go/types" "github.com/docker/compose-cli/api/compose" - "github.com/docker/compose-cli/utils" apitypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "golang.org/x/sync/errgroup" @@ -56,8 +55,7 @@ func (s *composeService) RunOneOffContainer(ctx context.Context, project *types. if err := s.waitDependencies(ctx, project, requestedService); err != nil { return "", err } - err := s.createContainer(ctx, project, requestedService, requestedService.ContainerName, 1) - if err != nil { + if err := s.createContainer(ctx, project, requestedService, requestedService.ContainerName, 1, opts.AutoRemove); err != nil { return "", err } @@ -69,19 +67,14 @@ func (s *composeService) RunOneOffContainer(ctx context.Context, project *types. containers, err := s.apiClient.ContainerList(ctx, apitypes.ContainerListOptions{ Filters: filters.NewArgs( - projectFilter(project.Name), + filters.Arg("label", fmt.Sprintf("%s=%s", slugLabel, slug)), ), All: true, }) if err != nil { return "", err } - var oneoffContainer apitypes.Container - for _, container := range containers { - if utils.StringContains(container.Names, "/"+containerID) { - oneoffContainer = container - } - } + oneoffContainer := containers[0] eg := errgroup.Group{} eg.Go(func() error { return s.attachContainerStreams(ctx, oneoffContainer, true, os.Stdin, os.Stdout) @@ -90,8 +83,7 @@ func (s *composeService) RunOneOffContainer(ctx context.Context, project *types. return "", err } - err = s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}) - if err != nil { + if err = s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}); err != nil { return "", err } err = eg.Wait() From 32d564493791a35a97b43e6f59944da29b0b865f Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Wed, 16 Dec 2020 18:50:03 +0100 Subject: [PATCH 16/19] Pass reader Writer as options to backend, remove hardcoded os.Stdout Signed-off-by: Guillaume Tardif --- aci/compose.go | 4 +-- api/client/compose.go | 4 +-- api/compose/api.go | 7 ++++-- cli/cmd/compose/run.go | 47 +++++++++++++++++++++--------------- ecs/local/compose.go | 4 +-- ecs/run.go | 4 +-- example/backend.go | 4 +-- local/compose/convergence.go | 2 +- local/compose/create.go | 2 +- local/compose/run.go | 27 ++++++++++----------- 10 files changed, 57 insertions(+), 48 deletions(-) diff --git a/aci/compose.go b/aci/compose.go index a8e65c2a..b3fe0210 100644 --- a/aci/compose.go +++ b/aci/compose.go @@ -202,6 +202,6 @@ func (cs *aciComposeService) Convert(ctx context.Context, project *types.Project return nil, errdefs.ErrNotImplemented } -func (cs *aciComposeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { - return "", errdefs.ErrNotImplemented +func (cs *aciComposeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) error { + return errdefs.ErrNotImplemented } diff --git a/api/client/compose.go b/api/client/compose.go index 2525b0d5..8f4c199a 100644 --- a/api/client/compose.go +++ b/api/client/compose.go @@ -72,6 +72,6 @@ func (c *composeService) Convert(context.Context, *types.Project, compose.Conver return nil, errdefs.ErrNotImplemented } -func (c *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { - return "", errdefs.ErrNotImplemented +func (c *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) error { + return errdefs.ErrNotImplemented } diff --git a/api/compose/api.go b/api/compose/api.go index daea325d..c6faa5c3 100644 --- a/api/compose/api.go +++ b/api/compose/api.go @@ -18,6 +18,7 @@ package compose import ( "context" + "io" "github.com/compose-spec/compose-go/types" ) @@ -47,7 +48,7 @@ type Service interface { // Convert translate compose model into backend's native format Convert(ctx context.Context, project *types.Project, options ConvertOptions) ([]byte, error) // RunOneOffContainer creates a service oneoff container and starts its dependencies - RunOneOffContainer(ctx context.Context, project *types.Project, opts RunOptions) (string, error) + RunOneOffContainer(ctx context.Context, project *types.Project, opts RunOptions) error } // UpOptions group options of the Up API @@ -68,12 +69,14 @@ type ConvertOptions struct { Format string } -// RunOptions holds all flags for compose run +// RunOptions options to execute compose run type RunOptions struct { Name string Command []string Detach bool AutoRemove bool + Writer io.Writer + Reader io.Reader } // PortPublisher hold status about published port diff --git a/cli/cmd/compose/run.go b/cli/cmd/compose/run.go index 437315fb..8b6f41c8 100644 --- a/cli/cmd/compose/run.go +++ b/cli/cmd/compose/run.go @@ -18,11 +18,12 @@ package compose import ( "context" - "fmt" + "os" "github.com/compose-spec/compose-go/types" "github.com/spf13/cobra" + "github.com/docker/compose-cli/api/client" "github.com/docker/compose-cli/api/compose" "github.com/docker/compose-cli/progress" ) @@ -74,20 +75,7 @@ func runRun(ctx context.Context, opts runOptions) error { originalServices := project.Services _, err = progress.Run(ctx, func(ctx context.Context) (string, error) { - dependencies := types.Services{} - for _, service := range originalServices { - if service.Name != opts.Name { - dependencies = append(dependencies, service) - } - } - project.Services = dependencies - if err := c.ComposeService().Create(ctx, project); err != nil { - return "", err - } - if err := c.ComposeService().Start(ctx, project, nil); err != nil { - return "", err - } - return "", nil + return "", startDependencies(ctx, c, project, opts.Name) }) if err != nil { return err @@ -95,13 +83,32 @@ func runRun(ctx context.Context, opts runOptions) error { project.Services = originalServices // start container and attach to container streams - runOpts := compose.RunOptions{Name: opts.Name, Command: opts.Command, Detach: opts.Detach, AutoRemove: opts.Remove} - containerID, err := c.ComposeService().RunOneOffContainer(ctx, project, runOpts) - if err != nil { + runOpts := compose.RunOptions{ + Name: opts.Name, + Command: opts.Command, + Detach: opts.Detach, + AutoRemove: opts.Remove, + Writer: os.Stdout, + Reader: os.Stdin, + } + return c.ComposeService().RunOneOffContainer(ctx, project, runOpts) +} + +func startDependencies(ctx context.Context, c *client.Client, project *types.Project, requestedService string) error { + originalServices := project.Services + dependencies := types.Services{} + for _, service := range originalServices { + if service.Name != requestedService { + dependencies = append(dependencies, service) + } + } + project.Services = dependencies + if err := c.ComposeService().Create(ctx, project); err != nil { return err } - if opts.Detach { - fmt.Printf("%s", containerID) + if err := c.ComposeService().Start(ctx, project, nil); err != nil { + return err } return nil + } diff --git a/ecs/local/compose.go b/ecs/local/compose.go index 8f5c5bd3..959f1c58 100644 --- a/ecs/local/compose.go +++ b/ecs/local/compose.go @@ -163,6 +163,6 @@ func (e ecsLocalSimulation) Ps(ctx context.Context, projectName string) ([]compo func (e ecsLocalSimulation) List(ctx context.Context, projectName string) ([]compose.Stack, error) { return e.compose.List(ctx, projectName) } -func (e ecsLocalSimulation) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { - return "", errors.Wrap(errdefs.ErrNotImplemented, "use docker-compose run") +func (e ecsLocalSimulation) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) error { + return errors.Wrap(errdefs.ErrNotImplemented, "use docker-compose run") } diff --git a/ecs/run.go b/ecs/run.go index 4ce83c33..6e3f8bf9 100644 --- a/ecs/run.go +++ b/ecs/run.go @@ -24,6 +24,6 @@ import ( "github.com/docker/compose-cli/errdefs" ) -func (b *ecsAPIService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { - return "", errdefs.ErrNotImplemented +func (b *ecsAPIService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) error { + return errdefs.ErrNotImplemented } diff --git a/example/backend.go b/example/backend.go index 2aba4a76..e0c62a6b 100644 --- a/example/backend.go +++ b/example/backend.go @@ -182,6 +182,6 @@ func (cs *composeService) Logs(ctx context.Context, projectName string, consumer func (cs *composeService) Convert(ctx context.Context, project *types.Project, options compose.ConvertOptions) ([]byte, error) { return nil, errdefs.ErrNotImplemented } -func (cs *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { - return "", errdefs.ErrNotImplemented +func (cs *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) error { + return errdefs.ErrNotImplemented } diff --git a/local/compose/convergence.go b/local/compose/convergence.go index 826251c6..8e23d4e9 100644 --- a/local/compose/convergence.go +++ b/local/compose/convergence.go @@ -229,7 +229,7 @@ func (s *composeService) restartContainer(ctx context.Context, container moby.Co } func (s *composeService) createMobyContainer(ctx context.Context, project *types.Project, service types.ServiceConfig, name string, number int, container *moby.Container, autoRemove bool) error { - containerConfig, hostConfig, networkingConfig, err := getContainerCreateOptions(project, service, number, container, autoRemove) + containerConfig, hostConfig, networkingConfig, err := getCreateOptions(project, service, number, container, autoRemove) if err != nil { return err } diff --git a/local/compose/create.go b/local/compose/create.go index 25897c49..b97a8173 100644 --- a/local/compose/create.go +++ b/local/compose/create.go @@ -91,7 +91,7 @@ func (s *composeService) ensureProjectVolumes(ctx context.Context, project *type return nil } -func getContainerCreateOptions(p *types.Project, s types.ServiceConfig, number int, inherit *moby.Container, autoRemove bool) (*container.Config, *container.HostConfig, *network.NetworkingConfig, error) { +func getCreateOptions(p *types.Project, s types.ServiceConfig, number int, inherit *moby.Container, autoRemove bool) (*container.Config, *container.HostConfig, *network.NetworkingConfig, error) { hash, err := jsonHash(s) if err != nil { return nil, nil, nil, err diff --git a/local/compose/run.go b/local/compose/run.go index 38260e1a..b9d590f4 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -19,7 +19,6 @@ package compose import ( "context" "fmt" - "os" "github.com/compose-spec/compose-go/types" "github.com/docker/compose-cli/api/compose" @@ -30,7 +29,7 @@ import ( moby "github.com/docker/docker/pkg/stringid" ) -func (s *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) (string, error) { +func (s *composeService) RunOneOffContainer(ctx context.Context, project *types.Project, opts compose.RunOptions) error { originalServices := project.Services var requestedService types.ServiceConfig for _, service := range originalServices { @@ -53,16 +52,20 @@ func (s *composeService) RunOneOffContainer(ctx context.Context, project *types. requestedService.Labels = requestedService.Labels.Add(oneoffLabel, "True") if err := s.waitDependencies(ctx, project, requestedService); err != nil { - return "", err + return err } if err := s.createContainer(ctx, project, requestedService, requestedService.ContainerName, 1, opts.AutoRemove); err != nil { - return "", err + return err } - containerID := requestedService.ContainerName if opts.Detach { - return containerID, s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}) + err := s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}) + if err != nil { + return err + } + fmt.Fprintln(opts.Writer, containerID) + return nil } containers, err := s.apiClient.ContainerList(ctx, apitypes.ContainerListOptions{ @@ -72,20 +75,16 @@ func (s *composeService) RunOneOffContainer(ctx context.Context, project *types. All: true, }) if err != nil { - return "", err + return err } oneoffContainer := containers[0] eg := errgroup.Group{} eg.Go(func() error { - return s.attachContainerStreams(ctx, oneoffContainer, true, os.Stdin, os.Stdout) + return s.attachContainerStreams(ctx, oneoffContainer, true, opts.Reader, opts.Writer) }) - if err != nil { - return "", err - } if err = s.apiClient.ContainerStart(ctx, containerID, apitypes.ContainerStartOptions{}); err != nil { - return "", err + return err } - err = eg.Wait() - return containerID, err + return eg.Wait() } From dc211b178ad647d30c207324ec5510e927852c84 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Thu, 17 Dec 2020 10:31:47 +0100 Subject: [PATCH 17/19] Fixed race when down with multiple containers for one service Signed-off-by: Guillaume Tardif --- local/compose/down.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/local/compose/down.go b/local/compose/down.go index 0227ddb2..704f832c 100644 --- a/local/compose/down.go +++ b/local/compose/down.go @@ -91,16 +91,17 @@ func (s *composeService) Down(ctx context.Context, projectName string, options c func (s *composeService) removeContainers(ctx context.Context, w progress.Writer, eg *errgroup.Group, containers []moby.Container) error { for _, container := range containers { + toDelete := container eg.Go(func() error { - eventName := "Container " + getContainerName(container) + eventName := "Container " + getContainerName(toDelete) w.Event(progress.StoppingEvent(eventName)) - err := s.apiClient.ContainerStop(ctx, container.ID, nil) + err := s.apiClient.ContainerStop(ctx, toDelete.ID, nil) if err != nil { w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping")) return err } w.Event(progress.RemovingEvent(eventName)) - err = s.apiClient.ContainerRemove(ctx, container.ID, moby.ContainerRemoveOptions{}) + err = s.apiClient.ContainerRemove(ctx, toDelete.ID, moby.ContainerRemoveOptions{}) if err != nil { w.Event(progress.ErrorMessageEvent(eventName, "Error while Removing")) return err From db73c928f38553708eea061fef28581b951791a6 Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Thu, 17 Dec 2020 11:57:37 +0100 Subject: [PATCH 18/19] Ensure image exist for run service Signed-off-by: Guillaume Tardif --- local/compose/run.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/local/compose/run.go b/local/compose/run.go index b9d590f4..0f5e2564 100644 --- a/local/compose/run.go +++ b/local/compose/run.go @@ -51,6 +51,9 @@ func (s *composeService) RunOneOffContainer(ctx context.Context, project *types. requestedService.Labels = requestedService.Labels.Add(slugLabel, slug) requestedService.Labels = requestedService.Labels.Add(oneoffLabel, "True") + if err := s.ensureImagesExists(ctx, project); err != nil { // all dependencies already checked, but might miss requestedService img + return err + } if err := s.waitDependencies(ctx, project, requestedService); err != nil { return err } From 3714ab7d41af64f456e110e572681d570a598c1d Mon Sep 17 00:00:00 2001 From: Guillaume Tardif Date: Thu, 17 Dec 2020 12:43:39 +0100 Subject: [PATCH 19/19] Attaching container logs Signed-off-by: Guillaume Tardif --- local/compose/attach.go | 1 + tests/compose-e2e/compose_test.go | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/local/compose/attach.go b/local/compose/attach.go index fac55eed..3866cf9a 100644 --- a/local/compose/attach.go +++ b/local/compose/attach.go @@ -120,6 +120,7 @@ func (s *composeService) getContainerStreams(ctx context.Context, container moby Stdin: true, Stdout: true, Stderr: true, + Logs: true, }) if err != nil { return nil, nil, err diff --git a/tests/compose-e2e/compose_test.go b/tests/compose-e2e/compose_test.go index 8ef6e9d4..3c8117c5 100644 --- a/tests/compose-e2e/compose_test.go +++ b/tests/compose-e2e/compose_test.go @@ -108,7 +108,8 @@ func TestLocalComposeRun(t *testing.T) { t.Run("compose run", func(t *testing.T) { res := c.RunDockerCmd("compose", "run", "-f", "./fixtures/run-test/docker-compose.yml", "back") - res.Assert(t, icmd.Expected{Out: "Hello there!!"}) + lines := Lines(res.Stdout()) + assert.Equal(t, lines[len(lines)-1], "Hello there!!", res.Stdout()) }) t.Run("check run container exited", func(t *testing.T) { @@ -142,7 +143,7 @@ func TestLocalComposeRun(t *testing.T) { t.Run("compose run --rm", func(t *testing.T) { res := c.RunDockerCmd("compose", "run", "-f", "./fixtures/run-test/docker-compose.yml", "--rm", "back", "/bin/sh", "-c", "echo Hello again") lines := Lines(res.Stdout()) - assert.Equal(t, lines[len(lines)-1], "Hello again") + assert.Equal(t, lines[len(lines)-1], "Hello again", res.Stdout()) }) t.Run("check run container removed", func(t *testing.T) { @@ -151,7 +152,7 @@ func TestLocalComposeRun(t *testing.T) { }) t.Run("down", func(t *testing.T) { - _ = c.RunDockerCmd("compose", "down", "-f", "./fixtures/run-test/docker-compose.yml") + c.RunDockerCmd("compose", "down", "-f", "./fixtures/run-test/docker-compose.yml") res := c.RunDockerCmd("ps", "--all") assert.Assert(t, !strings.Contains(res.Stdout(), "run-test"), res.Stdout()) })