From 953a7a3f4c8de879947a838a7cb63e7dd5935470 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 10 Apr 2020 16:09:03 +0200 Subject: [PATCH 001/205] Initial commit Signed-off-by: Nicolas De Loof From 1312eec07728db069c868d59d89136136906c42a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 10 Apr 2020 16:10:08 +0200 Subject: [PATCH 002/205] Project skaffloding Signed-off-by: Nicolas De Loof --- ecs/LICENSE | 191 ++++++++++++++++++++++++++++++++++++++++++++++++++ ecs/README.md | 1 + ecs/go.mod | 3 + 3 files changed, 195 insertions(+) create mode 100644 ecs/LICENSE create mode 100644 ecs/README.md create mode 100644 ecs/go.mod diff --git a/ecs/LICENSE b/ecs/LICENSE new file mode 100644 index 00000000..6d8d58fb --- /dev/null +++ b/ecs/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2013-2018 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ecs/README.md b/ecs/README.md new file mode 100644 index 00000000..128f6e0c --- /dev/null +++ b/ecs/README.md @@ -0,0 +1 @@ +# Docker CLI plugin for Amazon ECS diff --git a/ecs/go.mod b/ecs/go.mod new file mode 100644 index 00000000..96009881 --- /dev/null +++ b/ecs/go.mod @@ -0,0 +1,3 @@ +module github.com/docker/ecs-plugin + +go 1.13 From ba6c599de227c27f0efab0d49662e83958d669e4 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 14 Apr 2020 08:40:52 +0200 Subject: [PATCH 003/205] This is a CLI plugin Signed-off-by: Nicolas De Loof --- ecs/Makefile | 10 + ecs/cmd/main/main.go | 69 +++++++ ecs/go.mod | 47 +++++ ecs/go.sum | 400 +++++++++++++++++++++++++++++++++++++++ ecs/pkg/amazon/client.go | 7 + ecs/pkg/compose/opts.go | 16 ++ 6 files changed, 549 insertions(+) create mode 100644 ecs/Makefile create mode 100644 ecs/cmd/main/main.go create mode 100644 ecs/go.sum create mode 100644 ecs/pkg/amazon/client.go create mode 100644 ecs/pkg/compose/opts.go diff --git a/ecs/Makefile b/ecs/Makefile new file mode 100644 index 00000000..d8cc4a50 --- /dev/null +++ b/ecs/Makefile @@ -0,0 +1,10 @@ +build: + go build -v -o dist/docker-ecs cmd/main/main.go + +test: ## Run tests + go test ./... -v + +dev: build + ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" + +.PHONY: build test dev \ No newline at end of file diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go new file mode 100644 index 00000000..e248b0c7 --- /dev/null +++ b/ecs/cmd/main/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + "github.com/docker/cli/cli-plugins/manager" + "github.com/docker/cli/cli-plugins/plugin" + "github.com/docker/cli/cli/command" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/spf13/cobra" +) + +const version = "0.0.1" + +func main() { + plugin.Run(func(dockerCli command.Cli) *cobra.Command { + cmd := NewRootCmd("ecs", dockerCli) + return cmd + }, manager.Metadata{ + SchemaVersion: "0.1.0", + Vendor: "Docker Inc.", + Version: version, + Experimental: true, + }) +} + +type clusterOptions struct { + region string + cluster string +} + +// NewRootCmd returns the base root command. +func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { + var opts clusterOptions + + cmd := &cobra.Command{ + Short: "Docker ECS", + Long: `run multi-container applications on Amazon ECS.`, + Use: name, + Annotations: map[string]string{"experimentalCLI": "true"}, + } + cmd.AddCommand( + VersionCommand(), + ComposeCommand(&opts), + ) + cmd.Flags().StringVarP(&opts.cluster, "cluster", "c", "default", "ECS cluster") + cmd.Flags().StringVarP(&opts.region, "region", "r", "", "AWS region") + + return cmd +} + +func VersionCommand() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Show version.", + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Printf("Docker ECS plugin %s\n", version) + return nil + }, + } +} + +func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "compose", + } + opts := &compose.ProjectOptions{} + opts.AddFlags(cmd.Flags()) + return cmd +} diff --git a/ecs/go.mod b/ecs/go.mod index 96009881..573b02b4 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -1,3 +1,50 @@ module github.com/docker/ecs-plugin +require ( + github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect + github.com/Microsoft/hcsshim v0.8.7 // indirect + github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect + github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect + github.com/aws/aws-sdk-go v1.28.9 + github.com/bitly/go-hostpool v0.1.0 // indirect + github.com/bitly/go-simplejson v0.5.0 // indirect + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/bugsnag/bugsnag-go v1.5.3 // indirect + github.com/bugsnag/panicwrap v1.2.0 // indirect + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/cloudflare/cfssl v1.4.1 // indirect + github.com/compose-spec/compose-go v0.0.0-20200409090215-53c0040c9127 + github.com/containerd/containerd v1.3.2 // indirect + github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect + github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/docker v1.4.2-0.20200128034134-2ebaeef943cc // indirect + github.com/docker/docker-credential-helpers v0.6.3 // indirect + github.com/docker/go v1.5.1-1 // indirect + github.com/docker/go-metrics v0.0.1 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect + github.com/go-sql-driver/mysql v1.5.0 // indirect + github.com/gofrs/uuid v3.2.0+incompatible // indirect + github.com/gogo/protobuf v1.3.1 // indirect + github.com/gorilla/mux v1.7.3 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect + github.com/jinzhu/gorm v1.9.12 // indirect + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect + github.com/lib/pq v1.3.0 // indirect + github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect + github.com/miekg/pkcs11 v1.0.3 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/spf13/cobra v0.0.5 + github.com/spf13/pflag v1.0.3 + github.com/theupdateframework/notary v0.6.1 // indirect + github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + google.golang.org/grpc v1.27.0 // indirect + gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect + gopkg.in/fatih/pool.v2 v2.0.0 // indirect + gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect + vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect +) + go 1.13 diff --git a/ecs/go.sum b/ecs/go.sum new file mode 100644 index 00000000..3bc9036d --- /dev/null +++ b/ecs/go.sum @@ -0,0 +1,400 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c/go.mod h1:hSVuE3qU7grINVSwrmzHfpg9k87ALBk+XaualNyUzI4= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= +github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= +github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= +github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0= +github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= +github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bugsnag/bugsnag-go v1.5.3 h1:yeRUT3mUE13jL1tGwvoQsKdVbAsQx9AJ+fqahKveP04= +github.com/bugsnag/bugsnag-go v1.5.3/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= +github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiKw= +github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo= +github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= +github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= +github.com/compose-spec/compose-go v0.0.0-20200131085702-0b38cc2d8e6b h1:VK0c2Hfrg9FHjvJpWfGwiHPP2UeU0QZ6/5/dN0ehbSQ= +github.com/compose-spec/compose-go v0.0.0-20200131085702-0b38cc2d8e6b/go.mod h1:KoJjdV81vERSyYVuQD63nryyt8ZTlqTWe8JuJIMhRo4= +github.com/compose-spec/compose-go v0.0.0-20200409090215-53c0040c9127 h1:mAsQN3s19glh3KBOQjiRYBhqaX1SdzNqhB3/cuqgSbE= +github.com/compose-spec/compose-go v0.0.0-20200409090215-53c0040c9127/go.mod h1:1PUpzRF1O/65VOqXZuwpCuYY7pJxbIq1jbAvAf62FGM= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2 h1:ForxmXkA6tPIvffbrDAcPUIB32QgXkt2XFj+F0UxetA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb h1:nXPkFq8X1a9ycY3GYQpFNxHh3j2JgY7zDZfq2EXMIzk= +github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 h1:FwssHbCDJD025h+BchanCwE1Q8fyMgqDr2mOQAWOLGw= +github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20200128034134-2ebaeef943cc h1:2xtQXEoAs2hjCs4Ez4/KT2mDaYrXwcUi7TCkfyT+n2k= +github.com/docker/docker v1.4.2-0.20200128034134-2ebaeef943cc/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/go v1.5.1-1 h1:hr4w35acWBPhGBXlzPoHpmZ/ygPjnmFVxGxxGnMyP7k= +github.com/docker/go v1.5.1-1/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE= +github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q= +github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= +github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/kisom/goutils v1.1.0/go.mod h1:+UBTfd78habUYWFbNWTJNG+jNG/i/lGURakr4A/yNRw= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c= +github.com/lib/pq v0.0.0-20180201184707-88edab080323/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/theupdateframework/notary v0.6.1 h1:7wshjstgS9x9F5LuB1L5mBI2xNMObWqjz+cjWoom6l0= +github.com/theupdateframework/notary v0.6.1/go.mod h1:MOfgIfmox8s7/7fduvB2xyPPMJCrjRLRizA8OFwpnKY= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= +github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg2K1HrLU= +github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= +github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcrypto v0.0.0-20190729165852-9051775e6a2e h1:mvOa4+/DXStR4ZXOks/UsjeFdn5O5JpLUtzqk9U8xXw= +github.com/zmap/zcrypto v0.0.0-20190729165852-9051775e6a2e/go.mod h1:w7kd3qXHh8FNaczNjslXqvFQiv5mMWRXlL9klTUAHc8= +github.com/zmap/zlint v0.0.0-20190806154020-fd021b4cfbeb h1:vxqkjztXSaPVDc8FQCdHTaejm2x747f6yPbnu1h2xkg= +github.com/zmap/zlint v0.0.0-20190806154020-fd021b4cfbeb/go.mod h1:29UiAJNsiVdvTBFCJW8e3q6dcDbOoPkhMgttOSCIMMY= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYRuq8JQ1aa7LJt8EXVyo= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/dancannon/gorethink.v3 v3.0.5 h1:/g7PWP7zUS6vSNmHSDbjCHQh1Rqn8Jy6zSMQxAsBSMQ= +gopkg.in/dancannon/gorethink.v3 v3.0.5/go.mod h1:GXsi1e3N2OcKhcP6nsYABTiUejbWMFO4GY5a4pEaeEc= +gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= +gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y2O4MU= +gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.0 h1:d+tVGRu6X0ZBQ+kyAR8JKi6AXhTP2gmQaoIYaGFz634= +gotest.tools/v3 v3.0.0/go.mod h1:TUP+/YtXl/dp++T+SZ5v2zUmLVBHmptSb/ajDLCJ+3c= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= +vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go new file mode 100644 index 00000000..ea788b0b --- /dev/null +++ b/ecs/pkg/amazon/client.go @@ -0,0 +1,7 @@ +package amazon + +import "github.com/aws/aws-sdk-go/aws/session" + +type Client struct { + sess *session.Session +} diff --git a/ecs/pkg/compose/opts.go b/ecs/pkg/compose/opts.go new file mode 100644 index 00000000..6755abc2 --- /dev/null +++ b/ecs/pkg/compose/opts.go @@ -0,0 +1,16 @@ +package compose + +import ( + _ "github.com/compose-spec/compose-go/types" + "github.com/spf13/pflag" +) + +type ProjectOptions struct { + ConfigPaths []string + name string +} + +func (o *ProjectOptions) AddFlags(flags *pflag.FlagSet) { + flags.StringArrayVarP(&o.ConfigPaths, "file", "f", nil, "Specify an alternate compose file") + flags.StringVarP(&o.name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") +} From 40bf8c2dae64bb2dad78e241188e70f7df738d3a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 14 Apr 2020 11:42:04 +0200 Subject: [PATCH 004/205] Load a compose file and pass Project to cobra command close #2 Signed-off-by: Nicolas De Loof --- ecs/go.mod | 2 + ecs/go.sum | 7 + ecs/pkg/compose/opts.go | 16 +- ecs/pkg/compose/project.go | 165 ++++++++++++++++++ ecs/pkg/compose/project_test.go | 45 +++++ .../simple/compose-with-overrides.yaml | 4 + ecs/pkg/compose/testdata/simple/compose.yaml | 4 + 7 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 ecs/pkg/compose/project.go create mode 100644 ecs/pkg/compose/project_test.go create mode 100644 ecs/pkg/compose/testdata/simple/compose-with-overrides.yaml create mode 100644 ecs/pkg/compose/testdata/simple/compose.yaml diff --git a/ecs/go.mod b/ecs/go.mod index 573b02b4..1189b1d5 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -35,6 +35,7 @@ require ( github.com/miekg/pkcs11 v1.0.3 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/sirupsen/logrus v1.5.0 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 github.com/theupdateframework/notary v0.6.1 // indirect @@ -44,6 +45,7 @@ require ( gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect + gotest.tools/v3 v3.0.2 vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect ) diff --git a/ecs/go.sum b/ecs/go.sum index 3bc9036d..f2d87a8c 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -140,6 +140,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -180,6 +181,7 @@ github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -192,6 +194,7 @@ github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -280,9 +283,12 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg2K1HrLU= github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= @@ -392,6 +398,7 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.0 h1:d+tVGRu6X0ZBQ+kyAR8JKi6AXhTP2gmQaoIYaGFz634= gotest.tools/v3 v3.0.0/go.mod h1:TUP+/YtXl/dp++T+SZ5v2zUmLVBHmptSb/ajDLCJ+3c= +gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/ecs/pkg/compose/opts.go b/ecs/pkg/compose/opts.go index 6755abc2..9429fb01 100644 --- a/ecs/pkg/compose/opts.go +++ b/ecs/pkg/compose/opts.go @@ -1,7 +1,7 @@ package compose import ( - _ "github.com/compose-spec/compose-go/types" + "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -14,3 +14,17 @@ func (o *ProjectOptions) AddFlags(flags *pflag.FlagSet) { flags.StringArrayVarP(&o.ConfigPaths, "file", "f", nil, "Specify an alternate compose file") flags.StringVarP(&o.name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") } + + +type ProjectFunc func(project *Project, args []string) error + +// WithProject wrap a ProjectFunc into a cobra command +func WithProject(options *ProjectOptions, f ProjectFunc) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + project, err := projectFromOptions(options) + if err != nil { + return err + } + return f(project, args) + } +} diff --git a/ecs/pkg/compose/project.go b/ecs/pkg/compose/project.go new file mode 100644 index 00000000..c2c6c007 --- /dev/null +++ b/ecs/pkg/compose/project.go @@ -0,0 +1,165 @@ +package compose + +import ( + "fmt" + "github.com/compose-spec/compose-go/loader" + "github.com/compose-spec/compose-go/types" + "github.com/sirupsen/logrus" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" +) + +type Project struct { + types.Config + projectDir string + Name string `yaml:"-" json:"-"` +} + +func NewProject(config types.ConfigDetails, name string) (*Project, error) { + model, err := loader.Load(config) + if err != nil { + return nil, err + } + + p := Project{ + Config: *model, + projectDir: config.WorkingDir, + Name: name, + } + return &p, nil +} + + +// projectFromOptions load a compose project based on command line options +func projectFromOptions(options *ProjectOptions) (*Project, error) { + configPath, err := getConfigPathFromOptions(options) + if err != nil { + return nil, err + } + + name := options.name + if name == "" { + name = os.Getenv("COMPOSE_PROJECT_NAME") + } + + workingDir := filepath.Dir(configPath[0]) + + if name == "" { + r := regexp.MustCompile(`[^a-z0-9\\-_]+`) + name = r.ReplaceAllString(strings.ToLower(filepath.Base(workingDir)), "") + } + + configs, err := parseConfigs(configPath) + if err != nil { + return nil, err + } + + return NewProject(types.ConfigDetails{ + WorkingDir: workingDir, + ConfigFiles: configs, + Environment: environment(), + }, name) +} + +func getConfigPathFromOptions(options *ProjectOptions) ([]string, error) { + paths := []string{} + pwd, err := os.Getwd() + if err != nil { + return nil, err + } + + if len(options.ConfigPaths) != 0 { + for _, f := range options.ConfigPaths { + if f == "-" { + paths = append(paths, f) + continue + } + if !filepath.IsAbs(f) { + f = filepath.Join(pwd, f) + } + if _, err := os.Stat(f); err != nil { + return nil, err + } + paths = append(paths, f) + } + return paths, nil + } + + sep := os.Getenv("COMPOSE_FILE_SEPARATOR") + if sep == "" { + sep = string(os.PathListSeparator) + } + f := os.Getenv("COMPOSE_FILE") + if f != "" { + return strings.Split(f, sep), nil + } + + for { + candidates := []string{} + for _, n := range SupportedFilenames { + f := filepath.Join(pwd, n) + if _, err := os.Stat(f); err == nil { + candidates = append(candidates, f) + } + } + if len(candidates) > 0 { + winner := candidates[0] + if len(candidates) > 1 { + logrus.Warnf("Found multiple config files with supported names: %s", strings.Join(candidates, ", ")) + logrus.Warnf("Using %s\n", winner) + } + return []string{winner}, nil + } + parent := filepath.Dir(pwd) + if parent == pwd { + return nil, fmt.Errorf("Can't find a suitable configuration file in this directory or any parent. Are you in the right directory?") + } + pwd = parent + } +} + +var SupportedFilenames = []string{"compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"} + +func parseConfigs(configPaths []string) ([]types.ConfigFile, error) { + files := []types.ConfigFile{} + for _, f := range configPaths { + var ( + b []byte + err error + ) + if f == "-" { + b, err = ioutil.ReadAll(os.Stdin) + } else { + if _, err := os.Stat(f); err != nil { + return nil, err + } + b, err = ioutil.ReadFile(f) + } + if err != nil { + return nil, err + } + config, err := loader.ParseYAML(b) + if err != nil { + return nil, err + } + files = append(files, types.ConfigFile{Filename: f, Config: config}) + } + return files, nil +} + +func environment() map[string]string { + return getAsEqualsMap(os.Environ()) +} + +// getAsEqualsMap split key=value formatted strings into a key : value map +func getAsEqualsMap(em []string) map[string]string { + m := make(map[string]string) + for _, v := range em { + kv := strings.SplitN(v, "=", 2) + m[kv[0]] = kv[1] + } + return m +} diff --git a/ecs/pkg/compose/project_test.go b/ecs/pkg/compose/project_test.go new file mode 100644 index 00000000..d5f77140 --- /dev/null +++ b/ecs/pkg/compose/project_test.go @@ -0,0 +1,45 @@ +package compose + +import ( + "gotest.tools/v3/assert" + "os" + "testing" +) + +func Test_project_name(t *testing.T) { + p, err := projectFromOptions(&ProjectOptions{ + name: "my_project", + ConfigPaths: []string{"testdata/simple/compose.yaml"}, + }) + assert.NilError(t, err) + assert.Equal(t, p.Name, "my_project") + + p, err = projectFromOptions(&ProjectOptions{ + name: "", + ConfigPaths: []string{"testdata/simple/compose.yaml"}, + }) + assert.NilError(t, err) + assert.Equal(t, p.Name, "simple") + + os.Setenv("COMPOSE_PROJECT_NAME", "my_project_from_env") + p, err = projectFromOptions(&ProjectOptions{ + name: "", + ConfigPaths: []string{"testdata/simple/compose.yaml"}, + }) + assert.NilError(t, err) + assert.Equal(t, p.Name, "my_project_from_env") +} + +func Test_project_from_set_of_files(t *testing.T) { + p, err := projectFromOptions(&ProjectOptions{ + name: "my_project", + ConfigPaths: []string{ + "testdata/simple/compose.yaml", + "testdata/simple/compose-with-overrides.yaml", + }, + }) + assert.NilError(t, err) + service, err := p.GetService("simple") + assert.NilError(t, err) + assert.Equal(t, service.Image, "haproxy") +} diff --git a/ecs/pkg/compose/testdata/simple/compose-with-overrides.yaml b/ecs/pkg/compose/testdata/simple/compose-with-overrides.yaml new file mode 100644 index 00000000..3dc8a0b6 --- /dev/null +++ b/ecs/pkg/compose/testdata/simple/compose-with-overrides.yaml @@ -0,0 +1,4 @@ +version: "3" +services: + simple: + image: haproxy diff --git a/ecs/pkg/compose/testdata/simple/compose.yaml b/ecs/pkg/compose/testdata/simple/compose.yaml new file mode 100644 index 00000000..4b3f9af2 --- /dev/null +++ b/ecs/pkg/compose/testdata/simple/compose.yaml @@ -0,0 +1,4 @@ +version: "3" +services: + simple: + image: nginx \ No newline at end of file From 91daf0dcc0de009fbc9386b5a665a906519cb20a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 14 Apr 2020 17:44:00 +0200 Subject: [PATCH 005/205] Skeletton for "compose up" command Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 21 +++++++++++++++++++++ ecs/pkg/amazon/client.go | 34 ++++++++++++++++++++++++++++++++-- ecs/pkg/amazon/compose.go | 11 +++++++++++ ecs/pkg/compose/api.go | 5 +++++ 4 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 ecs/pkg/amazon/compose.go create mode 100644 ecs/pkg/compose/api.go diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index e248b0c7..83d13021 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -5,6 +5,7 @@ import ( "github.com/docker/cli/cli-plugins/manager" "github.com/docker/cli/cli-plugins/plugin" "github.com/docker/cli/cli/command" + "github.com/docker/ecs-plugin/pkg/amazon" "github.com/docker/ecs-plugin/pkg/compose" "github.com/spf13/cobra" ) @@ -24,6 +25,7 @@ func main() { } type clusterOptions struct { + profile string region string cluster string } @@ -42,6 +44,7 @@ func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { VersionCommand(), ComposeCommand(&opts), ) + cmd.Flags().StringVarP(&opts.profile, "profile", "p", "default", "AWS Profile") cmd.Flags().StringVarP(&opts.cluster, "cluster", "c", "default", "ECS cluster") cmd.Flags().StringVarP(&opts.region, "region", "r", "", "AWS region") @@ -65,5 +68,23 @@ func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { } opts := &compose.ProjectOptions{} opts.AddFlags(cmd.Flags()) + + cmd.AddCommand( + UpCommand(clusteropts, opts), + ) return cmd } + +func UpCommand(clusteropts *clusterOptions, opts *compose.ProjectOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "up", + RunE: compose.WithProject(opts, func(project *compose.Project, args []string) error { + client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) + if err != nil { + return err + } + return client.ComposeUp(project) + }), + } + return cmd +} \ No newline at end of file diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index ea788b0b..e1f5d2e9 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -1,7 +1,37 @@ package amazon -import "github.com/aws/aws-sdk-go/aws/session" +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/docker/ecs-plugin/pkg/compose" +) -type Client struct { + +const ( + ProjectTag = "com.docker.compose.project" +) + +func NewClient(profile string, cluster string, region string) (compose.API, error) { + sess, err := session.NewSessionWithOptions(session.Options{ + Profile: profile, + Config: aws.Config{ + Region: aws.String(region), + }, + }) + if err != nil { + return nil, err + } + return &client{ + Cluster: cluster, + Region: region, + sess: sess, + }, nil +} + +type client struct { + Cluster string + Region string sess *session.Session } + +var _ compose.API = &client{} diff --git a/ecs/pkg/amazon/compose.go b/ecs/pkg/amazon/compose.go new file mode 100644 index 00000000..6a107379 --- /dev/null +++ b/ecs/pkg/amazon/compose.go @@ -0,0 +1,11 @@ +package amazon + +import ( + "fmt" + "github.com/docker/ecs-plugin/pkg/compose" +) + +func (c *client) ComposeUp(project *compose.Project) error { + fmt.Println("TODO Up") + return nil +} diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go new file mode 100644 index 00000000..14e1ca05 --- /dev/null +++ b/ecs/pkg/compose/api.go @@ -0,0 +1,5 @@ +package compose + +type API interface { + ComposeUp(project *Project) error +} \ No newline at end of file From 4542e05ddfe0182c648e45d56b3e15e0bc63a94d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 14 Apr 2020 18:03:33 +0200 Subject: [PATCH 006/205] API calls to register services matching compose.yaml Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/client.go | 12 ++++++ ecs/pkg/amazon/compose.go | 72 ++++++++++++++++++++++++++++++- ecs/pkg/amazon/ecs.go | 21 +++++++++ ecs/pkg/amazon/logs.go | 26 +++++++++++ ecs/pkg/amazon/network.go | 90 +++++++++++++++++++++++++++++++++++++++ ecs/pkg/amazon/roles.go | 48 +++++++++++++++++++++ 6 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 ecs/pkg/amazon/ecs.go create mode 100644 ecs/pkg/amazon/logs.go create mode 100644 ecs/pkg/amazon/network.go create mode 100644 ecs/pkg/amazon/roles.go diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index e1f5d2e9..8e98a55f 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -3,6 +3,10 @@ package amazon import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/aws/aws-sdk-go/service/iam" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -25,6 +29,10 @@ func NewClient(profile string, cluster string, region string) (compose.API, erro Cluster: cluster, Region: region, sess: sess, + ECS: ecs.New(sess), + EC2: ec2.New(sess), + CW: cloudwatchlogs.New(sess), + IAM: iam.New(sess), }, nil } @@ -32,6 +40,10 @@ type client struct { Cluster string Region string sess *session.Session + ECS *ecs.ECS + EC2 *ec2.EC2 + CW *cloudwatchlogs.CloudWatchLogs + IAM *iam.IAM } var _ compose.API = &client{} diff --git a/ecs/pkg/amazon/compose.go b/ecs/pkg/amazon/compose.go index 6a107379..202143f7 100644 --- a/ecs/pkg/amazon/compose.go +++ b/ecs/pkg/amazon/compose.go @@ -1,11 +1,79 @@ package amazon import ( - "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" ) func (c *client) ComposeUp(project *compose.Project) error { - fmt.Println("TODO Up") + vpc, err := c.GetDefaultVPC() + if err != nil { + return err + } + subnets, err := c.GetSubNets(vpc) + if err != nil { + return err + } + + securityGroup, err := c.CreateSecurityGroup(project, vpc) + if err != nil { + return err + } + + logGroup, err := c.GetOrCreateLogGroup(project.Name) + if err != nil { + return err + } + + for _, service := range project.Services { + err = c.CreateService(service, securityGroup, subnets, logGroup) + if err != nil { + return err + } + } return nil } + +func (c *client) CreateService(service types.ServiceConfig, securityGroup *string, subnets []*string, logGroup *string) error { + task, err := ConvertToTaskDefinition(service) + if err != nil { + return err + } + + role, err := c.GetEcsTaskExecutionRole(service) + if err != nil { + return err + } + + task.ExecutionRoleArn = role + + for _, def := range task.ContainerDefinitions { + def.LogConfiguration.Options["awslogs-group"] = logGroup + def.LogConfiguration.Options["awslogs-stream-prefix"] = aws.String(service.Name) + def.LogConfiguration.Options["awslogs-region"] = aws.String(c.Region) + } + + arn, err := c.RegisterTaskDefinition(task) + if err != nil { + return err + } + + _, err = c.ECS.CreateService(&ecs.CreateServiceInput{ + Cluster: aws.String(c.Cluster), + DesiredCount: aws.Int64(1), // FIXME get from deploy options + LaunchType: aws.String(ecs.LaunchTypeFargate), //FIXME use service.Isolation tro select EC2 vs Fargate + NetworkConfiguration: &ecs.NetworkConfiguration{ + AwsvpcConfiguration: &ecs.AwsVpcConfiguration{ + AssignPublicIp: aws.String(ecs.AssignPublicIpEnabled), + SecurityGroups: []*string{securityGroup}, + Subnets: subnets, + }, + }, + ServiceName: aws.String(service.Name), + SchedulingStrategy: aws.String(ecs.SchedulingStrategyReplica), + TaskDefinition: arn, + }) + return err +} diff --git a/ecs/pkg/amazon/ecs.go b/ecs/pkg/amazon/ecs.go new file mode 100644 index 00000000..c405b135 --- /dev/null +++ b/ecs/pkg/amazon/ecs.go @@ -0,0 +1,21 @@ +package amazon + +import ( + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/compose-spec/compose-go/types" + "github.com/sirupsen/logrus" +) + +func ConvertToTaskDefinition(service types.ServiceConfig) (*ecs.RegisterTaskDefinitionInput, error) { + panic("Please implement me") +} + + +func (c client) RegisterTaskDefinition(task *ecs.RegisterTaskDefinitionInput) (*string, error) { + logrus.Debug("Register Task Definition") + def, err := c.ECS.RegisterTaskDefinition(task) + if err != nil { + return nil, err + } + return def.TaskDefinition.TaskDefinitionArn, err +} diff --git a/ecs/pkg/amazon/logs.go b/ecs/pkg/amazon/logs.go new file mode 100644 index 00000000..7553ba1b --- /dev/null +++ b/ecs/pkg/amazon/logs.go @@ -0,0 +1,26 @@ +package amazon + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/sirupsen/logrus" +) + +// GetOrCreateLogGroup retrieve a pre-existing log group for project or create one +func (c client) GetOrCreateLogGroup(project string) (*string, error) { + logrus.Debug("Create Log Group") + logGroup := fmt.Sprintf("/ecs/%s", project) + _, err := c.CW.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{ + LogGroupName: aws.String(logGroup), + Tags: map[string]*string{ + ProjectTag: aws.String(project), + }, + }) + if err != nil { + if _, ok := err.(*cloudwatchlogs.ResourceAlreadyExistsException); !ok { + return nil, err + } + } + return &logGroup, nil +} diff --git a/ecs/pkg/amazon/network.go b/ecs/pkg/amazon/network.go new file mode 100644 index 00000000..d994cce1 --- /dev/null +++ b/ecs/pkg/amazon/network.go @@ -0,0 +1,90 @@ +package amazon + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/sirupsen/logrus" +) + +// GetDefaultVPC retrieve the default VPC for AWS account +func (c client) GetDefaultVPC() (*string, error) { + logrus.Debug("Retrieve default VPC") + vpcs, err := c.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("isDefault"), + Values: []*string{aws.String("true")}, + }, + }, + }) + if err != nil { + return nil, err + } + if len(vpcs.Vpcs) == 0 { + return nil, fmt.Errorf("account has not default VPC") + } + return vpcs.Vpcs[0].VpcId, nil +} + + +// GetSubNets retrieve default subnets for a VPC +func (c client) GetSubNets(vpc *string) ([]*string, error) { + logrus.Debug("Retrieve SubNets") + subnets, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ + DryRun: nil, + Filters: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{vpc}, + }, + { + Name: aws.String("default-for-az"), + Values: []*string{aws.String("true")}, + }, + }, + }) + if err != nil { + return nil, err + } + + ids := []*string{} + for _, subnet := range subnets.Subnets { + ids = append(ids, subnet.SubnetId) + } + return ids, nil +} + +// CreateSecurityGroup create a security group for the project +func (c client) CreateSecurityGroup(project *compose.Project, vpc *string) (*string, error) { + logrus.Debug("Create Security Group") + name := fmt.Sprintf("%s Security Group", project) + securityGroup, err := c.EC2.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ + Description: aws.String(name), + GroupName: aws.String(name), + VpcId: vpc, + }) + if err != nil { + return nil, err + } + + _, err = c.EC2.CreateTags(&ec2.CreateTagsInput{ + Resources: []*string{securityGroup.GroupId}, + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(name), + }, + { + Key: aws.String(ProjectTag), + Value: aws.String(project.Name), + }, + }, + }) + if err != nil { + return nil, err + } + + return securityGroup.GroupId, nil +} \ No newline at end of file diff --git a/ecs/pkg/amazon/roles.go b/ecs/pkg/amazon/roles.go new file mode 100644 index 00000000..3e8c5303 --- /dev/null +++ b/ecs/pkg/amazon/roles.go @@ -0,0 +1,48 @@ +package amazon + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/compose-spec/compose-go/types" + "github.com/sirupsen/logrus" +) + +const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + +var defaultTaskExecutionRole *string + +// GetEcsTaskExecutionRole retrieve the role ARN to apply for task execution +func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (*string, error) { + if arn, ok := spec.Extras["x-ecs-TaskExecutionRole"]; ok { + s := arn.(string) + return &s, nil + } + if defaultTaskExecutionRole != nil { + return defaultTaskExecutionRole, nil + } + + logrus.Debug("Retrieve Task Execution Role") + entities, err := c.IAM.ListEntitiesForPolicy(&iam.ListEntitiesForPolicyInput{ + EntityFilter: aws.String("Role"), + PolicyArn: aws.String(ECSTaskExecutionPolicy), + }) + if err != nil { + return nil, err + } + if len(entities.PolicyRoles) == 0 { + return nil, fmt.Errorf("no Role is attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") + } + if len(entities.PolicyRoles) > 1 { + return nil, fmt.Errorf("multiple Roles are attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") + } + + role, err := c.IAM.GetRole(&iam.GetRoleInput{ + RoleName: entities.PolicyRoles[0].RoleName, + }) + if err != nil { + return nil, err + } + defaultTaskExecutionRole = role.Role.Arn + return role.Role.Arn, nil +} From fc7266f3f78824c184ec9fb6d097645e788fae1f Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 15 Apr 2020 10:38:19 +0200 Subject: [PATCH 007/205] Convert compose service into TaskDefinition (code imported from prototype) Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/compose.go | 29 ++- ecs/pkg/amazon/ecs.go | 5 - ecs/pkg/amazon/logs.go | 7 +- ecs/pkg/amazon/network.go | 25 ++- ecs/pkg/convert/convert.go | 358 +++++++++++++++++++++++++++++++++++++ 5 files changed, 406 insertions(+), 18 deletions(-) create mode 100644 ecs/pkg/convert/convert.go diff --git a/ecs/pkg/amazon/compose.go b/ecs/pkg/amazon/compose.go index 202143f7..c8b4352b 100644 --- a/ecs/pkg/amazon/compose.go +++ b/ecs/pkg/amazon/compose.go @@ -5,6 +5,8 @@ import ( "github.com/aws/aws-sdk-go/service/ecs" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/convert" + "github.com/sirupsen/logrus" ) func (c *client) ComposeUp(project *compose.Project) error { @@ -22,13 +24,13 @@ func (c *client) ComposeUp(project *compose.Project) error { return err } - logGroup, err := c.GetOrCreateLogGroup(project.Name) + logGroup, err := c.GetOrCreateLogGroup(project) if err != nil { return err } for _, service := range project.Services { - err = c.CreateService(service, securityGroup, subnets, logGroup) + _, err = c.CreateService(project, service, securityGroup, subnets, logGroup) if err != nil { return err } @@ -36,15 +38,15 @@ func (c *client) ComposeUp(project *compose.Project) error { return nil } -func (c *client) CreateService(service types.ServiceConfig, securityGroup *string, subnets []*string, logGroup *string) error { - task, err := ConvertToTaskDefinition(service) +func (c *client) CreateService(project *compose.Project, service types.ServiceConfig, securityGroup *string, subnets []*string, logGroup *string) (*string, error) { + task, err := convert.Convert(project, service) if err != nil { - return err + return nil, err } role, err := c.GetEcsTaskExecutionRole(service) if err != nil { - return err + return nil, err } task.ExecutionRoleArn = role @@ -57,10 +59,11 @@ func (c *client) CreateService(service types.ServiceConfig, securityGroup *strin arn, err := c.RegisterTaskDefinition(task) if err != nil { - return err + return nil, err } - _, err = c.ECS.CreateService(&ecs.CreateServiceInput{ + logrus.Debug("Create Service") + created, err := c.ECS.CreateService(&ecs.CreateServiceInput{ Cluster: aws.String(c.Cluster), DesiredCount: aws.Int64(1), // FIXME get from deploy options LaunchType: aws.String(ecs.LaunchTypeFargate), //FIXME use service.Isolation tro select EC2 vs Fargate @@ -75,5 +78,13 @@ func (c *client) CreateService(service types.ServiceConfig, securityGroup *strin SchedulingStrategy: aws.String(ecs.SchedulingStrategyReplica), TaskDefinition: arn, }) - return err + + for _, port := range service.Ports { + err = c.ExposePort(securityGroup, port) + if err != nil { + return nil, err + } + } + + return created.Service.ServiceArn, err } diff --git a/ecs/pkg/amazon/ecs.go b/ecs/pkg/amazon/ecs.go index c405b135..5366a7a0 100644 --- a/ecs/pkg/amazon/ecs.go +++ b/ecs/pkg/amazon/ecs.go @@ -2,14 +2,9 @@ package amazon import ( "github.com/aws/aws-sdk-go/service/ecs" - "github.com/compose-spec/compose-go/types" "github.com/sirupsen/logrus" ) -func ConvertToTaskDefinition(service types.ServiceConfig) (*ecs.RegisterTaskDefinitionInput, error) { - panic("Please implement me") -} - func (c client) RegisterTaskDefinition(task *ecs.RegisterTaskDefinitionInput) (*string, error) { logrus.Debug("Register Task Definition") diff --git a/ecs/pkg/amazon/logs.go b/ecs/pkg/amazon/logs.go index 7553ba1b..b42bc8b0 100644 --- a/ecs/pkg/amazon/logs.go +++ b/ecs/pkg/amazon/logs.go @@ -4,17 +4,18 @@ import ( "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" ) // GetOrCreateLogGroup retrieve a pre-existing log group for project or create one -func (c client) GetOrCreateLogGroup(project string) (*string, error) { +func (c client) GetOrCreateLogGroup(project *compose.Project) (*string, error) { logrus.Debug("Create Log Group") - logGroup := fmt.Sprintf("/ecs/%s", project) + logGroup := fmt.Sprintf("/ecs/%s", project.Name) _, err := c.CW.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{ LogGroupName: aws.String(logGroup), Tags: map[string]*string{ - ProjectTag: aws.String(project), + ProjectTag: aws.String(project.Name), }, }) if err != nil { diff --git a/ecs/pkg/amazon/network.go b/ecs/pkg/amazon/network.go index d994cce1..a876b4bc 100644 --- a/ecs/pkg/amazon/network.go +++ b/ecs/pkg/amazon/network.go @@ -4,8 +4,10 @@ import ( "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" + "strings" ) // GetDefaultVPC retrieve the default VPC for AWS account @@ -59,7 +61,7 @@ func (c client) GetSubNets(vpc *string) ([]*string, error) { // CreateSecurityGroup create a security group for the project func (c client) CreateSecurityGroup(project *compose.Project, vpc *string) (*string, error) { logrus.Debug("Create Security Group") - name := fmt.Sprintf("%s Security Group", project) + name := fmt.Sprintf("%s Security Group", project.Name) securityGroup, err := c.EC2.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ Description: aws.String(name), GroupName: aws.String(name), @@ -87,4 +89,25 @@ func (c client) CreateSecurityGroup(project *compose.Project, vpc *string) (*str } return securityGroup.GroupId, nil +} + + +func (c *client) ExposePort(securityGroup *string, port types.ServicePortConfig) error { + logrus.Debugf("Authorize ingress port %d/%s\n", port.Published, port.Protocol) + _, err := c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: securityGroup, + IpPermissions: []*ec2.IpPermission{ + { + IpProtocol: aws.String(strings.ToUpper(port.Protocol)), + IpRanges: []*ec2.IpRange{ + { + CidrIp: aws.String("0.0.0.0/0"), + }, + }, + FromPort: aws.Int64(int64(port.Target)), + ToPort: aws.Int64(int64(port.Target)), + }, + }, + }) + return err } \ No newline at end of file diff --git a/ecs/pkg/convert/convert.go b/ecs/pkg/convert/convert.go new file mode 100644 index 00000000..502f5089 --- /dev/null +++ b/ecs/pkg/convert/convert.go @@ -0,0 +1,358 @@ +package convert + +import ( + "github.com/docker/ecs-plugin/pkg/compose" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/compose-spec/compose-go/types" + "github.com/docker/cli/opts" +) + +func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.RegisterTaskDefinitionInput, error) { + _, err := toCPULimits(service) + if err != nil { + return nil, err + } + + foo := int64(256) + logDriver := "awslogs" // FIXME could be set by service.Logging, especially to enable use of firelens + return &ecs.RegisterTaskDefinitionInput{ + ContainerDefinitions: []*ecs.ContainerDefinition{ + // Here we can declare sidecars and init-containers using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_dependson + { + Command: toStringPtrSlice(service.Command), + Cpu: &foo, + DependsOn: nil, + DisableNetworking: toBoolPtr(service.NetworkMode == "none"), + DnsSearchDomains: toStringPtrSlice(service.DNSSearch), + DnsServers: toStringPtrSlice(service.DNS), + DockerLabels: nil, + DockerSecurityOptions: toStringPtrSlice(service.SecurityOpt), + EntryPoint: toStringPtrSlice(service.Entrypoint), + Environment: toKeyValuePairPtr(service.Environment), + Essential: toBoolPtr(true), + ExtraHosts: toHostEntryPtr(service.ExtraHosts), + FirelensConfiguration: nil, + HealthCheck: toHealthCheck(service.HealthCheck), + Hostname: toStringPtr(service.Hostname), + Image: toStringPtr(service.Image), + Interactive: nil, + Links: nil, + LinuxParameters: toLinuxParameters(service), + LogConfiguration: &ecs.LogConfiguration{ + LogDriver: &logDriver, + Options: map[string]*string{}, + SecretOptions: nil, + }, + Memory: toMemoryLimits(service.Deploy), + MemoryReservation: toMemoryReservation(service.Deploy), + MountPoints: nil, + Name: toStringPtr(service.Name), + PortMappings: toPortMappings(service.Ports), + Privileged: toBoolPtr(service.Privileged), + PseudoTerminal: toBoolPtr(service.Tty), + ReadonlyRootFilesystem: toBoolPtr(service.ReadOnly), + RepositoryCredentials: nil, + ResourceRequirements: nil, + Secrets: nil, + StartTimeout: nil, + StopTimeout: durationToInt64Ptr(service.StopGracePeriod), + SystemControls: nil, + Ulimits: toUlimits(service.Ulimits), + User: toStringPtr(service.User), + VolumesFrom: nil, + WorkingDirectory: toStringPtr(service.WorkingDir), + }, + }, + Cpu: toCPU(service), + ExecutionRoleArn: nil, + Family: toStringPtr(project.Name), + IpcMode: toStringPtr(service.Ipc), + Memory: toMemory(service), + NetworkMode: toStringPtr("awsvpc"), // FIXME could be set by service.NetworkMode, Fargate only supports network mode ‘awsvpc’. + PidMode: toStringPtr(service.Pid), + PlacementConstraints: toPlacementConstraints(service.Deploy), + ProxyConfiguration: nil, + RequiresCompatibilities: toRequiresCompatibilities(ecs.LaunchTypeFargate), + Tags: nil, + Volumes: []*ecs.Volume{ + { + /* ONLY supported when using EC2 launch type + DockerVolumeConfiguration: { + Autoprovision: nil, + Driver: nil, + DriverOpts: nil, + Labels: nil, + Scope: nil, + }, */ + /* Beta and ONLY supported when using EC2 launch type + EfsVolumeConfiguration: { + FileSystemId: nil, + RootDirectory: nil, + }, */ + /* Bind mount host volume + Host: { + SourcePath: + }, */ + Name: aws.String("MyVolume"), + }, + }, + }, nil + +} + +func toCPU(service types.ServiceConfig) *string { + // FIXME based on service's memory/cpu requirements, select the adequate Fargate CPU + v := "256" + return &v +} + +func toMemory(service types.ServiceConfig) *string { + // FIXME based on service's memory/cpu requirements, select the adequate Fargate CPU + v := "512" + return &v +} + +func toCPULimits(service types.ServiceConfig) (*int64, error) { + if service.Deploy == nil { + return nil, nil + } + res := service.Deploy.Resources.Limits + if res == nil { + return nil, nil + } + if res.NanoCPUs == "" { + return nil, nil + } + v, err := opts.ParseCPUs(res.NanoCPUs) + if err != nil { + return nil, err + } + return &v, nil +} + +func toRequiresCompatibilities(isolation string) []*string { + if isolation == "" { + return nil + } + return []*string{&isolation} +} + +func hasMemoryOrMemoryReservation(service types.ServiceConfig) bool { + if service.Deploy == nil { + return false + } + if service.Deploy.Resources.Reservations != nil { + return true + } + if service.Deploy.Resources.Limits != nil { + return true + } + return false +} + +func toPlacementConstraints(deploy *types.DeployConfig) []*ecs.TaskDefinitionPlacementConstraint { + if deploy == nil || deploy.Placement.Constraints == nil || len(deploy.Placement.Constraints) == 0 { + return nil + } + pl := []*ecs.TaskDefinitionPlacementConstraint{} + for _, c := range deploy.Placement.Constraints { + pl = append(pl, &ecs.TaskDefinitionPlacementConstraint{ + Expression: toStringPtr(c), + Type: nil, + }) + } + return pl +} + +func toPortMappings(ports []types.ServicePortConfig) []*ecs.PortMapping { + if len(ports) == 0 { + return nil + } + m := []*ecs.PortMapping{} + for _, p := range ports { + m = append(m, &ecs.PortMapping{ + ContainerPort: uint32Toint64Ptr(p.Target), + HostPort: uint32Toint64Ptr(p.Published), + Protocol: toStringPtr(p.Protocol), + }) + } + return m +} + +func toUlimits(ulimits map[string]*types.UlimitsConfig) []*ecs.Ulimit { + if len(ulimits) == 0 { + return nil + } + u := []*ecs.Ulimit{} + for k, v := range ulimits { + u = append(u, &ecs.Ulimit{ + Name: toStringPtr(k), + SoftLimit: intToInt64Ptr(v.Soft), + HardLimit: intToInt64Ptr(v.Hard), + }) + } + return u +} + +func uint32Toint64Ptr(i uint32) *int64 { + v := int64(i) + return &v +} + +func intToInt64Ptr(i int) *int64 { + v := int64(i) + return &v +} + +const Mb = 1024 * 1024 + +func toMemoryLimits(deploy *types.DeployConfig) *int64 { + if deploy == nil { + return nil + } + res := deploy.Resources.Limits + if res == nil { + return nil + } + v := int64(res.MemoryBytes) / Mb + return &v +} + +func toMemoryReservation(deploy *types.DeployConfig) *int64 { + if deploy == nil { + return nil + } + res := deploy.Resources.Reservations + if res == nil { + return nil + } + v := int64(res.MemoryBytes) / Mb + return &v +} + +func toLinuxParameters(service types.ServiceConfig) *ecs.LinuxParameters { + return &ecs.LinuxParameters{ + Capabilities: toKernelCapabilities(service.CapAdd, service.CapDrop), + Devices: nil, + InitProcessEnabled: service.Init, + MaxSwap: nil, + // FIXME SharedMemorySize: service.ShmSize, + Swappiness: nil, + Tmpfs: toTmpfs(service.Tmpfs), + } +} + +func toTmpfs(tmpfs types.StringList) []*ecs.Tmpfs { + if tmpfs == nil || len(tmpfs) == 0 { + return nil + } + o := []*ecs.Tmpfs{} + for _, t := range tmpfs { + path := t + o = append(o, &ecs.Tmpfs{ + ContainerPath: &path, + MountOptions: nil, + Size: nil, + }) + } + return o +} + +func toKernelCapabilities(add []string, drop []string) *ecs.KernelCapabilities { + if len(add) == 0 && len(drop) == 0 { + return nil + } + return &ecs.KernelCapabilities{ + Add: toStringPtrSlice(add), + Drop: toStringPtrSlice(drop), + } + +} + +func toHealthCheck(check *types.HealthCheckConfig) *ecs.HealthCheck { + if check == nil { + return nil + } + return &ecs.HealthCheck{ + Command: toStringPtrSlice(check.Test), + Interval: durationToInt64Ptr(check.Interval), + Retries: uint64ToInt64Ptr(check.Retries), + StartPeriod: durationToInt64Ptr(check.StartPeriod), + Timeout: durationToInt64Ptr(check.Timeout), + } +} + +func uint64ToInt64Ptr(i *uint64) *int64 { + if i == nil { + return nil + } + v := int64(*i) + return &v +} + +func durationToInt64Ptr(interval *types.Duration) *int64 { + if interval == nil { + return nil + } + v := int64(time.Duration(*interval).Seconds()) + return &v +} + +func toHostEntryPtr(hosts types.HostsList) []*ecs.HostEntry { + if hosts == nil || len(hosts) == 0 { + return nil + } + e := []*ecs.HostEntry{} + for _, h := range hosts { + host := h + e = append(e, &ecs.HostEntry{ + Hostname: &host, + }) + } + return e +} + +func toKeyValuePairPtr(environment types.MappingWithEquals) []*ecs.KeyValuePair { + if environment == nil || len(environment) == 0 { + return nil + } + pairs := []*ecs.KeyValuePair{} + for k, v := range environment { + name := k + value := v + pairs = append(pairs, &ecs.KeyValuePair{ + Name: &name, + Value: value, + }) + } + return pairs +} + +func toStringPtr(s string) *string { + if s == "" { + return nil + } + return &s +} + +func toStringPtrSlice(s []string) []*string { + if len(s) == 0 { + return nil + } + v := []*string{} + for _, x := range s { + value := x + v = append(v, &value) + } + return v +} + +func toBoolPtr(b bool) *bool { + if !b { + return nil + } + return &b +} \ No newline at end of file From 7763de47eb85d32853408488a42d4aa14ed1b89d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 16 Apr 2020 10:07:28 +0200 Subject: [PATCH 008/205] Introduce "down" command Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 15 +++++++ ecs/pkg/amazon/down.go | 60 ++++++++++++++++++++++++++++ ecs/pkg/amazon/{compose.go => up.go} | 0 ecs/pkg/compose/api.go | 1 + 4 files changed, 76 insertions(+) create mode 100644 ecs/pkg/amazon/down.go rename ecs/pkg/amazon/{compose.go => up.go} (100%) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 83d13021..daf11ac0 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -71,6 +71,7 @@ func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { cmd.AddCommand( UpCommand(clusteropts, opts), + DownCommand(clusteropts, opts), ) return cmd } @@ -87,4 +88,18 @@ func UpCommand(clusteropts *clusterOptions, opts *compose.ProjectOptions) *cobra }), } return cmd +} + +func DownCommand(clusteropts *clusterOptions, opts *compose.ProjectOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "down", + RunE: compose.WithProject(opts, func(project *compose.Project, args []string) error { + client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) + if err != nil { + return err + } + return client.ComposeDown(project) + }), + } + return cmd } \ No newline at end of file diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go new file mode 100644 index 00000000..611d84f4 --- /dev/null +++ b/ecs/pkg/amazon/down.go @@ -0,0 +1,60 @@ +package amazon + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/sirupsen/logrus" +) + +func (c *client) ComposeDown(project *compose.Project) error { + services := []*string{} + // FIXME we should be able to retrieve services by tags, so we don't need the initial compose file to run "down" + for _, service := range project.Services { + logrus.Debugf("Deleting service %q\n", service.Name) + out, err := c.ECS.DeleteService(&ecs.DeleteServiceInput{ + // Force to true so that we don't have to scale down to 0 + // before deleting + Force: aws.Bool(true), + Cluster: aws.String(c.Cluster), + Service: aws.String(service.Name), + }) + if err != nil { + return err + } + + logrus.Debugf("Service deleted %q\n", *out.Service.ServiceName) + services = append(services, out.Service.ServiceName) + } + logrus.Info("All services stopped") + + err := c.ECS.WaitUntilServicesInactive(&ecs.DescribeServicesInput{ + Services: services, + }) + if err != nil { + return err + } + + logrus.Debug("Deleting security groups") + groups, err := c.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("tag:" + ProjectTag), + Values: aws.StringSlice([]string{project.Name}), + }, + }, + }) + if err != nil { + return err + } + for _, g := range groups.SecurityGroups { + _, err = c.EC2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{ + GroupId: g.GroupId, + }) + if err != nil { + return err + } + } + return nil +} diff --git a/ecs/pkg/amazon/compose.go b/ecs/pkg/amazon/up.go similarity index 100% rename from ecs/pkg/amazon/compose.go rename to ecs/pkg/amazon/up.go diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 14e1ca05..cd6eb787 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -2,4 +2,5 @@ package compose type API interface { ComposeUp(project *Project) error + ComposeDown(project *Project) error } \ No newline at end of file From 17f3ff9db1f8b54ea93e7c3482dfddec81d48c9d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 16 Apr 2020 11:16:50 +0200 Subject: [PATCH 009/205] Convert services into TaskDefinition before creating resources close #6 Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/roles.go | 2 +- ecs/pkg/amazon/up.go | 27 +++++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/ecs/pkg/amazon/roles.go b/ecs/pkg/amazon/roles.go index 3e8c5303..bfa5026a 100644 --- a/ecs/pkg/amazon/roles.go +++ b/ecs/pkg/amazon/roles.go @@ -13,7 +13,7 @@ const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTa var defaultTaskExecutionRole *string // GetEcsTaskExecutionRole retrieve the role ARN to apply for task execution -func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (*string, error) { +func (c client) GetEcsTaskExecutionRole(spec *types.ServiceConfig) (*string, error) { if arn, ok := spec.Extras["x-ecs-TaskExecutionRole"]; ok { s := arn.(string) return &s, nil diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index c8b4352b..e54aae14 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -10,6 +10,22 @@ import ( ) func (c *client) ComposeUp(project *compose.Project) error { + type mapping struct { + service *types.ServiceConfig + task *ecs.RegisterTaskDefinitionInput + } + mappings := []mapping{} + for _, service := range project.Services { + task, err := convert.Convert(project, service) + if err != nil { + return err + } + mappings = append(mappings, mapping{ + service: &service, + task: task, + }) + } + vpc, err := c.GetDefaultVPC() if err != nil { return err @@ -29,8 +45,8 @@ func (c *client) ComposeUp(project *compose.Project) error { return err } - for _, service := range project.Services { - _, err = c.CreateService(project, service, securityGroup, subnets, logGroup) + for _, mapping := range mappings { + _, err = c.CreateService(project, mapping.service, mapping.task, securityGroup, subnets, logGroup) if err != nil { return err } @@ -38,12 +54,7 @@ func (c *client) ComposeUp(project *compose.Project) error { return nil } -func (c *client) CreateService(project *compose.Project, service types.ServiceConfig, securityGroup *string, subnets []*string, logGroup *string) (*string, error) { - task, err := convert.Convert(project, service) - if err != nil { - return nil, err - } - +func (c *client) CreateService(project *compose.Project, service *types.ServiceConfig, task *ecs.RegisterTaskDefinitionInput, securityGroup *string, subnets []*string, logGroup *string) (*string, error) { role, err := c.GetEcsTaskExecutionRole(service) if err != nil { return nil, err From a44ee2a4ed71c7abe24a1b701ec1cb28a9d49452 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 16 Apr 2020 15:15:39 +0200 Subject: [PATCH 010/205] Expose services using a LoadBalancer Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 4 +- ecs/pkg/amazon/client.go | 3 + ecs/pkg/amazon/down.go | 14 ++-- ecs/pkg/amazon/loadBalancer.go | 133 +++++++++++++++++++++++++++++++++ ecs/pkg/amazon/up.go | 30 +++++++- 5 files changed, 175 insertions(+), 9 deletions(-) create mode 100644 ecs/pkg/amazon/loadBalancer.go diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index daf11ac0..75b6c2b7 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -76,10 +76,10 @@ func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { return cmd } -func UpCommand(clusteropts *clusterOptions, opts *compose.ProjectOptions) *cobra.Command { +func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "up", - RunE: compose.WithProject(opts, func(project *compose.Project, args []string) error { + RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) if err != nil { return err diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index 8e98a55f..247536a9 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -6,6 +6,7 @@ import ( "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ecs" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/iam" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -31,6 +32,7 @@ func NewClient(profile string, cluster string, region string) (compose.API, erro sess: sess, ECS: ecs.New(sess), EC2: ec2.New(sess), + ELB: elbv2.New(sess), CW: cloudwatchlogs.New(sess), IAM: iam.New(sess), }, nil @@ -42,6 +44,7 @@ type client struct { sess *session.Session ECS *ecs.ECS EC2 *ec2.EC2 + ELB *elbv2.ELBV2 CW *cloudwatchlogs.CloudWatchLogs IAM *iam.IAM } diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index 611d84f4..e68c3132 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -9,6 +9,11 @@ import ( ) func (c *client) ComposeDown(project *compose.Project) error { + err := c.DeleteLoadBalancer(project) + if err != nil { + return err + } + services := []*string{} // FIXME we should be able to retrieve services by tags, so we don't need the initial compose file to run "down" for _, service := range project.Services { @@ -23,18 +28,17 @@ func (c *client) ComposeDown(project *compose.Project) error { if err != nil { return err } - - logrus.Debugf("Service deleted %q\n", *out.Service.ServiceName) - services = append(services, out.Service.ServiceName) + services = append(services, out.Service.ServiceArn) } - logrus.Info("All services stopped") - err := c.ECS.WaitUntilServicesInactive(&ecs.DescribeServicesInput{ + logrus.Info("Stopping services...") + err = c.ECS.WaitUntilServicesInactive(&ecs.DescribeServicesInput{ Services: services, }) if err != nil { return err } + logrus.Info("All services stopped") logrus.Debug("Deleting security groups") groups, err := c.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ diff --git a/ecs/pkg/amazon/loadBalancer.go b/ecs/pkg/amazon/loadBalancer.go new file mode 100644 index 00000000..6edc7955 --- /dev/null +++ b/ecs/pkg/amazon/loadBalancer.go @@ -0,0 +1,133 @@ +package amazon + +import ( + "fmt" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/sirupsen/logrus" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/compose-spec/compose-go/types" +) + +func (c client) CreateLoadBalancer(project *compose.Project, subnets []*string) (*string, error) { + logrus.Debug("Create Load Balancer") + alb, err := c.ELB.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + IpAddressType: nil, + Name: aws.String(fmt.Sprintf("%s-LoadBalancer", project.Name)), + Subnets: subnets, + Type: aws.String(elbv2.LoadBalancerTypeEnumNetwork), + Tags: []*elbv2.Tag{ + { + Key: aws.String("com.docker.compose.project"), + Value: aws.String(project.Name), + }, + }, + }) + if err != nil { + return nil, err + } + return alb.LoadBalancers[0].LoadBalancerArn, nil +} + +func (c client) DeleteLoadBalancer(project *compose.Project) error { + logrus.Debug("Delete Load Balancer") + // FIXME We can tag LoadBalancer but not search by tag ? + loadBalancer, err := c.ELB.DescribeLoadBalancers(&elbv2.DescribeLoadBalancersInput{ + Names: aws.StringSlice([]string{fmt.Sprintf("%s-LoadBalancer", project.Name)}), + }) + if err != nil { + return err + } + arn := loadBalancer.LoadBalancers[0].LoadBalancerArn + + err = c.DeleteListeners(arn) + if err != nil { + return err + } + + err = c.DeleteTargetGroups(arn) + if err != nil { + return err + } + + _, err = c.ELB.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{LoadBalancerArn: arn}) + return err +} + +func (c client) CreateTargetGroup(name string, vpc *string, port types.ServicePortConfig) (*string, error) { + logrus.Debugf("Create Target Group %d/%s\n", port.Target, port.Protocol) + group, err := c.ELB.CreateTargetGroup(&elbv2.CreateTargetGroupInput{ + Name: aws.String(name), + Port: aws.Int64(int64(port.Target)), + Protocol: aws.String(strings.ToUpper(port.Protocol)), + TargetType: aws.String("ip"), + VpcId: vpc, + }) + if err != nil { + return nil, err + } + arn := group.TargetGroups[0].TargetGroupArn + return arn, nil +} + +func (c client) DeleteTargetGroups(loadBalancer *string) error { + groups, err := c.ELB.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{ + LoadBalancerArn: loadBalancer, + }) + if err != nil { + return err + } + for _, group := range groups.TargetGroups { + logrus.Debugf("Delete Target Group %s\n", *group.TargetGroupArn) + _, err := c.ELB.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{ + TargetGroupArn: group.TargetGroupArn, + }) + if err != nil { + return err + } + } + return nil +} + +func (c client) CreateListener(port types.ServicePortConfig, arn *string, target *string) error { + logrus.Debugf("Create Listener %d\n", port.Published) + _, err := c.ELB.CreateListener(&elbv2.CreateListenerInput{ + DefaultActions: []*elbv2.Action{ + { + ForwardConfig: &elbv2.ForwardActionConfig{ + TargetGroups: []*elbv2.TargetGroupTuple{ + { + TargetGroupArn: target, + }, + }, + }, + Type: aws.String(elbv2.ActionTypeEnumForward), + }, + }, + LoadBalancerArn: arn, + Port: aws.Int64(int64(port.Published)), + Protocol: aws.String(strings.ToUpper(port.Protocol)), + }) + return err +} + +func (c client) DeleteListeners(loadBalancer *string) error { + listeners, err := c.ELB.DescribeListeners(&elbv2.DescribeListenersInput{ + LoadBalancerArn: loadBalancer, + }) + if err != nil { + return err + } + for _, listener := range listeners.Listeners { + logrus.Debugf("Delete Listener %s\n", *listener.ListenerArn) + _, err := c.ELB.DeleteListener(&elbv2.DeleteListenerInput{ + ListenerArn: listener.ListenerArn, + }) + if err != nil { + return err + } + } + return nil +} \ No newline at end of file diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index e54aae14..d0b84b2a 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -1,6 +1,7 @@ package amazon import ( + "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" "github.com/compose-spec/compose-go/types" @@ -40,13 +41,37 @@ func (c *client) ComposeUp(project *compose.Project) error { return err } + loadBalancer, err := c.CreateLoadBalancer(project, subnets) + if err != nil { + return err + } + logGroup, err := c.GetOrCreateLogGroup(project) if err != nil { return err } for _, mapping := range mappings { - _, err = c.CreateService(project, mapping.service, mapping.task, securityGroup, subnets, logGroup) + ingress := []*ecs.LoadBalancer{} + for _, port := range mapping.service.Ports { + name := fmt.Sprintf("%s-%s-%d-%s", project.Name, mapping.service.Name, port.Target, port.Protocol) + targetgroup, err := c.CreateTargetGroup(name, vpc, port) + if err != nil { + return err + } + ingress = append(ingress, &ecs.LoadBalancer{ + ContainerName: aws.String(mapping.service.Name), + ContainerPort: aws.Int64(int64(port.Target)), + TargetGroupArn: targetgroup, + }) + + err = c.CreateListener(port, loadBalancer, targetgroup) + if err != nil { + return err + } + } + + _, err = c.CreateService(project, mapping.service, mapping.task, securityGroup, subnets, logGroup, ingress) if err != nil { return err } @@ -54,7 +79,7 @@ func (c *client) ComposeUp(project *compose.Project) error { return nil } -func (c *client) CreateService(project *compose.Project, service *types.ServiceConfig, task *ecs.RegisterTaskDefinitionInput, securityGroup *string, subnets []*string, logGroup *string) (*string, error) { +func (c *client) CreateService(project *compose.Project, service *types.ServiceConfig, task *ecs.RegisterTaskDefinitionInput, securityGroup *string, subnets []*string, logGroup *string, ingress []*ecs.LoadBalancer) (*string, error) { role, err := c.GetEcsTaskExecutionRole(service) if err != nil { return nil, err @@ -88,6 +113,7 @@ func (c *client) CreateService(project *compose.Project, service *types.ServiceC ServiceName: aws.String(service.Name), SchedulingStrategy: aws.String(ecs.SchedulingStrategyReplica), TaskDefinition: arn, + LoadBalancers: ingress, }) for _, port := range service.Ports { From dd48cc4599a5319f9de63f0dacb42ba526005fb7 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 16 Apr 2020 16:18:06 +0200 Subject: [PATCH 011/205] Introduct option to re-use LoadBalancer Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 28 ++++++++++++++++++++++++---- ecs/pkg/amazon/down.go | 4 ++-- ecs/pkg/amazon/loadBalancer.go | 6 ++++-- ecs/pkg/amazon/up.go | 12 +++++++----- ecs/pkg/compose/api.go | 4 ++-- 5 files changed, 39 insertions(+), 15 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 75b6c2b7..18340476 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -76,7 +76,19 @@ func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { return cmd } +type upOptions struct { + loadBalancerArn string +} + +func (o upOptions) LoadBalancerArn() *string { + if o.loadBalancerArn == "" { + return nil + } + return &o.loadBalancerArn +} + func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { + opts := upOptions{} cmd := &cobra.Command{ Use: "up", RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { @@ -84,22 +96,30 @@ func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) if err != nil { return err } - return client.ComposeUp(project) + return client.ComposeUp(project, opts.LoadBalancerArn()) }), } + cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") return cmd } -func DownCommand(clusteropts *clusterOptions, opts *compose.ProjectOptions) *cobra.Command { +type downOptions struct { + KeepLoadBalancer bool +} + + +func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { + opts := downOptions{} cmd := &cobra.Command{ Use: "down", - RunE: compose.WithProject(opts, func(project *compose.Project, args []string) error { + RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) if err != nil { return err } - return client.ComposeDown(project) + return client.ComposeDown(project, opts.KeepLoadBalancer) }), } + cmd.Flags().BoolVar(&opts.KeepLoadBalancer, "keep-load-balancer", false, "Keep Load Balancer for further use") return cmd } \ No newline at end of file diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index e68c3132..dbd5b684 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -8,8 +8,8 @@ import ( "github.com/sirupsen/logrus" ) -func (c *client) ComposeDown(project *compose.Project) error { - err := c.DeleteLoadBalancer(project) +func (c *client) ComposeDown(project *compose.Project, keepLoadBalancer bool) error { + err := c.DeleteLoadBalancer(project, keepLoadBalancer) if err != nil { return err } diff --git a/ecs/pkg/amazon/loadBalancer.go b/ecs/pkg/amazon/loadBalancer.go index 6edc7955..6972406e 100644 --- a/ecs/pkg/amazon/loadBalancer.go +++ b/ecs/pkg/amazon/loadBalancer.go @@ -31,7 +31,7 @@ func (c client) CreateLoadBalancer(project *compose.Project, subnets []*string) return alb.LoadBalancers[0].LoadBalancerArn, nil } -func (c client) DeleteLoadBalancer(project *compose.Project) error { +func (c client) DeleteLoadBalancer(project *compose.Project, keepLoadBalancer bool) error { logrus.Debug("Delete Load Balancer") // FIXME We can tag LoadBalancer but not search by tag ? loadBalancer, err := c.ELB.DescribeLoadBalancers(&elbv2.DescribeLoadBalancersInput{ @@ -52,7 +52,9 @@ func (c client) DeleteLoadBalancer(project *compose.Project) error { return err } - _, err = c.ELB.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{LoadBalancerArn: arn}) + if !keepLoadBalancer { + _, err = c.ELB.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{LoadBalancerArn: arn}) + } return err } diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index d0b84b2a..0dbc35c9 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -10,7 +10,7 @@ import ( "github.com/sirupsen/logrus" ) -func (c *client) ComposeUp(project *compose.Project) error { +func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) error { type mapping struct { service *types.ServiceConfig task *ecs.RegisterTaskDefinitionInput @@ -41,9 +41,11 @@ func (c *client) ComposeUp(project *compose.Project) error { return err } - loadBalancer, err := c.CreateLoadBalancer(project, subnets) - if err != nil { - return err + if loadBalancerArn == nil { + loadBalancerArn, err = c.CreateLoadBalancer(project, subnets) + if err != nil { + return err + } } logGroup, err := c.GetOrCreateLogGroup(project) @@ -65,7 +67,7 @@ func (c *client) ComposeUp(project *compose.Project) error { TargetGroupArn: targetgroup, }) - err = c.CreateListener(port, loadBalancer, targetgroup) + err = c.CreateListener(port, loadBalancerArn, targetgroup) if err != nil { return err } diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index cd6eb787..ceebacb8 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -1,6 +1,6 @@ package compose type API interface { - ComposeUp(project *Project) error - ComposeDown(project *Project) error + ComposeUp(project *Project, loadBalancerArn *string) error + ComposeDown(project *Project, keepLoadBalancer bool) error } \ No newline at end of file From 4e72d1892a555360f06e8e78211a7751b2befa4c Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 20 Apr 2020 13:47:38 +0200 Subject: [PATCH 012/205] Prefer AWS API interface over actual implementation This will help introduce mock-based tests Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/client.go | 19 ++++++++++++++----- ecs/pkg/amazon/cloudformation.go | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 ecs/pkg/amazon/cloudformation.go diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index 247536a9..3dceb4de 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -3,11 +3,18 @@ package amazon import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/ecs" + "github.com/aws/aws-sdk-go/service/ecs/ecsiface" "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/iam/iamiface" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -35,6 +42,7 @@ func NewClient(profile string, cluster string, region string) (compose.API, erro ELB: elbv2.New(sess), CW: cloudwatchlogs.New(sess), IAM: iam.New(sess), + CF: cloudformation.New(sess), }, nil } @@ -42,11 +50,12 @@ type client struct { Cluster string Region string sess *session.Session - ECS *ecs.ECS - EC2 *ec2.EC2 - ELB *elbv2.ELBV2 - CW *cloudwatchlogs.CloudWatchLogs - IAM *iam.IAM + ECS ecsiface.ECSAPI + EC2 ec2iface.EC2API + ELB elbv2iface.ELBV2API + CW cloudwatchlogsiface.CloudWatchLogsAPI + IAM iamiface.IAMAPI + CF cloudformationiface.CloudFormationAPI } var _ compose.API = &client{} diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go new file mode 100644 index 00000000..1f74174f --- /dev/null +++ b/ecs/pkg/amazon/cloudformation.go @@ -0,0 +1 @@ +package amazon From b70f01d2f4d172ec537be41427c7f33c692d5005 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 21 Apr 2020 11:38:52 +0200 Subject: [PATCH 013/205] Adopt CloudFormation to create ECS app from compose.yaml Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 31 +++- ecs/go.mod | 1 + ecs/go.sum | 26 ++- ecs/pkg/amazon/client.go | 33 ++-- ecs/pkg/amazon/cloudformation.go | 80 +++++++++ ecs/pkg/amazon/down.go | 54 +----- ecs/pkg/amazon/ecs.go | 1 - ecs/pkg/amazon/loadBalancer.go | 5 +- ecs/pkg/amazon/logs.go | 1 + ecs/pkg/amazon/network.go | 13 +- ecs/pkg/amazon/roles.go | 3 +- ecs/pkg/amazon/up.go | 132 ++++----------- ecs/pkg/compose/api.go | 5 +- ecs/pkg/compose/opts.go | 1 - ecs/pkg/compose/project.go | 8 +- ecs/pkg/compose/project_test.go | 3 +- ecs/pkg/convert/convert.go | 278 +++++++++++++------------------ 17 files changed, 314 insertions(+), 361 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 18340476..cde9be3b 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/docker/cli/cli-plugins/manager" "github.com/docker/cli/cli-plugins/plugin" "github.com/docker/cli/cli/command" @@ -70,6 +71,7 @@ func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { opts.AddFlags(cmd.Flags()) cmd.AddCommand( + ConvertCommand(clusteropts, opts), UpCommand(clusteropts, opts), DownCommand(clusteropts, opts), ) @@ -87,6 +89,32 @@ func (o upOptions) LoadBalancerArn() *string { return &o.loadBalancerArn } +func ConvertCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { + opts := upOptions{} + cmd := &cobra.Command{ + Use: "convert", + RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) + if err != nil { + return err + } + template, err := client.Convert(project, opts.LoadBalancerArn()) + if err != nil { + return err + } + + j, err := template.JSON() + if err != nil { + fmt.Printf("Failed to generate JSON: %s\n", err) + } else { + fmt.Printf("%s\n", string(j)) + } + return nil + }), + } + return cmd +} + func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ @@ -107,7 +135,6 @@ type downOptions struct { KeepLoadBalancer bool } - func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { opts := downOptions{} cmd := &cobra.Command{ @@ -122,4 +149,4 @@ func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOption } cmd.Flags().BoolVar(&opts.KeepLoadBalancer, "keep-load-balancer", false, "Keep Load Balancer for further use") return cmd -} \ No newline at end of file +} diff --git a/ecs/go.mod b/ecs/go.mod index 1189b1d5..a8f444c5 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -6,6 +6,7 @@ require ( github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect github.com/aws/aws-sdk-go v1.28.9 + github.com/awslabs/goformation/v4 v4.8.0 github.com/bitly/go-hostpool v0.1.0 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect diff --git a/ecs/go.sum b/ecs/go.sum index f2d87a8c..dc5a0393 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -21,6 +21,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0= github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/awslabs/goformation/v4 v4.8.0 h1:UiUhyokRy3suEqBXTnipvY8klqY3Eyl4GCH17brraEc= +github.com/awslabs/goformation/v4 v4.8.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -46,8 +48,6 @@ github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiK github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo= github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= -github.com/compose-spec/compose-go v0.0.0-20200131085702-0b38cc2d8e6b h1:VK0c2Hfrg9FHjvJpWfGwiHPP2UeU0QZ6/5/dN0ehbSQ= -github.com/compose-spec/compose-go v0.0.0-20200131085702-0b38cc2d8e6b/go.mod h1:KoJjdV81vERSyYVuQD63nryyt8ZTlqTWe8JuJIMhRo4= github.com/compose-spec/compose-go v0.0.0-20200409090215-53c0040c9127 h1:mAsQN3s19glh3KBOQjiRYBhqaX1SdzNqhB3/cuqgSbE= github.com/compose-spec/compose-go v0.0.0-20200409090215-53c0040c9127/go.mod h1:1PUpzRF1O/65VOqXZuwpCuYY7pJxbIq1jbAvAf62FGM= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= @@ -139,7 +139,9 @@ github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1: github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -180,7 +182,6 @@ github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -205,8 +206,12 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= +github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.2.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -244,12 +249,14 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b h1:jUK33OXuZP/l6babJtnLo1qsGvq6G9so9KMflGAm4YA= +github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b/go.mod h1:8458kAagoME2+LN5//WxE71ysZ3B7r22fdgb7qVmXSY= +github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522 h1:fOCp11H0yuyAt2wqlbJtbyPzSgaxHTv8uN1pMpkG1t8= +github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522/go.mod h1:tQTYKOQgxoH3v6dEmdHiz4JG+nbxWwM5fgPQUpSZqVQ= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= @@ -285,9 +292,12 @@ github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= @@ -327,6 +337,8 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYR golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -385,10 +397,12 @@ gopkg.in/dancannon/gorethink.v3 v3.0.5 h1:/g7PWP7zUS6vSNmHSDbjCHQh1Rqn8Jy6zSMQxA gopkg.in/dancannon/gorethink.v3 v3.0.5/go.mod h1:GXsi1e3N2OcKhcP6nsYABTiUejbWMFO4GY5a4pEaeEc= gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y2O4MU= gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -396,8 +410,6 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.0 h1:d+tVGRu6X0ZBQ+kyAR8JKi6AXhTP2gmQaoIYaGFz634= -gotest.tools/v3 v3.0.0/go.mod h1:TUP+/YtXl/dp++T+SZ5v2zUmLVBHmptSb/ajDLCJ+3c= gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index 3dceb4de..ff202fc2 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -18,7 +18,6 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) - const ( ProjectTag = "com.docker.compose.project" ) @@ -35,27 +34,27 @@ func NewClient(profile string, cluster string, region string) (compose.API, erro } return &client{ Cluster: cluster, - Region: region, - sess: sess, - ECS: ecs.New(sess), - EC2: ec2.New(sess), - ELB: elbv2.New(sess), - CW: cloudwatchlogs.New(sess), - IAM: iam.New(sess), - CF: cloudformation.New(sess), + Region: region, + sess: sess, + ECS: ecs.New(sess), + EC2: ec2.New(sess), + ELB: elbv2.New(sess), + CW: cloudwatchlogs.New(sess), + IAM: iam.New(sess), + CF: cloudformation.New(sess), }, nil } type client struct { Cluster string - Region string - sess *session.Session - ECS ecsiface.ECSAPI - EC2 ec2iface.EC2API - ELB elbv2iface.ELBV2API - CW cloudwatchlogsiface.CloudWatchLogsAPI - IAM iamiface.IAMAPI - CF cloudformationiface.CloudFormationAPI + Region string + sess *session.Session + ECS ecsiface.ECSAPI + EC2 ec2iface.EC2API + ELB elbv2iface.ELBV2API + CW cloudwatchlogsiface.CloudWatchLogsAPI + IAM iamiface.IAMAPI + CF cloudformationiface.CloudFormationAPI } var _ compose.API = &client{} diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 1f74174f..0b3b0bc4 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -1 +1,81 @@ package amazon + +import ( + "fmt" + "strings" + + ecsapi "github.com/aws/aws-sdk-go/service/ecs" + "github.com/awslabs/goformation/v4/cloudformation" + "github.com/awslabs/goformation/v4/cloudformation/ec2" + "github.com/awslabs/goformation/v4/cloudformation/ecs" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/convert" +) + +func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*cloudformation.Template, error) { + template := cloudformation.NewTemplate() + + vpc, err := c.GetDefaultVPC() + if err != nil { + return nil, err + } + + subnets, err := c.GetSubNets(vpc) + if err != nil { + return nil, err + } + + var ingresses = []ec2.SecurityGroup_Ingress{} + for _, service := range project.Services { + for _, port := range service.Ports { + ingresses = append(ingresses, ec2.SecurityGroup_Ingress{ + CidrIp: "0.0.0.0/0", + Description: fmt.Sprintf("%d/%s", port.Target, port.Protocol), + FromPort: int(port.Target), + IpProtocol: strings.ToUpper(port.Protocol), + ToPort: int(port.Target), + }) + } + } + + securityGroup := fmt.Sprintf("%s Security Group", project.Name) + template.Resources["SecurityGroup"] = &ec2.SecurityGroup{ + GroupDescription: securityGroup, + GroupName: securityGroup, + SecurityGroupIngress: ingresses, + VpcId: *vpc, + } + + for _, service := range project.Services { + definition, err := convert.Convert(project, service) + if err != nil { + return nil, err + } + + role, err := c.GetEcsTaskExecutionRole(service) + if err != nil { + return nil, err + } + definition.TaskRoleArn = *role + + taskDefinition := fmt.Sprintf("%sTaskDefinition", service.Name) + template.Resources[taskDefinition] = definition + + template.Resources[service.Name] = &ecs.Service{ + Cluster: c.Cluster, + DesiredCount: 1, + LaunchType: ecsapi.LaunchTypeFargate, + NetworkConfiguration: &ecs.Service_NetworkConfiguration{ + AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ + AssignPublicIp: ecsapi.AssignPublicIpEnabled, + SecurityGroups: []string{cloudformation.Ref("SecurityGroup")}, + Subnets: subnets, + }, + }, + SchedulingStrategy: ecsapi.SchedulingStrategyReplica, + ServiceName: service.Name, + TaskDefinition: cloudformation.Ref(taskDefinition), + } + } + return template, nil +} diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index dbd5b684..ac264832 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -1,64 +1,18 @@ package amazon import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ecs" + "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" - "github.com/sirupsen/logrus" ) func (c *client) ComposeDown(project *compose.Project, keepLoadBalancer bool) error { - err := c.DeleteLoadBalancer(project, keepLoadBalancer) - if err != nil { - return err - } - - services := []*string{} - // FIXME we should be able to retrieve services by tags, so we don't need the initial compose file to run "down" - for _, service := range project.Services { - logrus.Debugf("Deleting service %q\n", service.Name) - out, err := c.ECS.DeleteService(&ecs.DeleteServiceInput{ - // Force to true so that we don't have to scale down to 0 - // before deleting - Force: aws.Bool(true), - Cluster: aws.String(c.Cluster), - Service: aws.String(service.Name), - }) - if err != nil { - return err - } - services = append(services, out.Service.ServiceArn) - } - - logrus.Info("Stopping services...") - err = c.ECS.WaitUntilServicesInactive(&ecs.DescribeServicesInput{ - Services: services, + _, err := c.CF.DeleteStack(&cloudformation.DeleteStackInput{ + StackName: &project.Name, }) if err != nil { return err } - logrus.Info("All services stopped") - logrus.Debug("Deleting security groups") - groups, err := c.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("tag:" + ProjectTag), - Values: aws.StringSlice([]string{project.Name}), - }, - }, - }) - if err != nil { - return err - } - for _, g := range groups.SecurityGroups { - _, err = c.EC2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{ - GroupId: g.GroupId, - }) - if err != nil { - return err - } - } + // TODO monitor progress return nil } diff --git a/ecs/pkg/amazon/ecs.go b/ecs/pkg/amazon/ecs.go index 5366a7a0..423980f3 100644 --- a/ecs/pkg/amazon/ecs.go +++ b/ecs/pkg/amazon/ecs.go @@ -5,7 +5,6 @@ import ( "github.com/sirupsen/logrus" ) - func (c client) RegisterTaskDefinition(task *ecs.RegisterTaskDefinitionInput) (*string, error) { logrus.Debug("Register Task Definition") def, err := c.ECS.RegisterTaskDefinition(task) diff --git a/ecs/pkg/amazon/loadBalancer.go b/ecs/pkg/amazon/loadBalancer.go index 6972406e..c8f96cd9 100644 --- a/ecs/pkg/amazon/loadBalancer.go +++ b/ecs/pkg/amazon/loadBalancer.go @@ -2,9 +2,10 @@ package amazon import ( "fmt" + "strings" + "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" - "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/elbv2" @@ -132,4 +133,4 @@ func (c client) DeleteListeners(loadBalancer *string) error { } } return nil -} \ No newline at end of file +} diff --git a/ecs/pkg/amazon/logs.go b/ecs/pkg/amazon/logs.go index b42bc8b0..0c06671c 100644 --- a/ecs/pkg/amazon/logs.go +++ b/ecs/pkg/amazon/logs.go @@ -2,6 +2,7 @@ package amazon import ( "fmt" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/docker/ecs-plugin/pkg/compose" diff --git a/ecs/pkg/amazon/network.go b/ecs/pkg/amazon/network.go index a876b4bc..26b41275 100644 --- a/ecs/pkg/amazon/network.go +++ b/ecs/pkg/amazon/network.go @@ -2,12 +2,13 @@ package amazon import ( "fmt" + "strings" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" - "strings" ) // GetDefaultVPC retrieve the default VPC for AWS account @@ -30,9 +31,8 @@ func (c client) GetDefaultVPC() (*string, error) { return vpcs.Vpcs[0].VpcId, nil } - // GetSubNets retrieve default subnets for a VPC -func (c client) GetSubNets(vpc *string) ([]*string, error) { +func (c client) GetSubNets(vpc *string) ([]string, error) { logrus.Debug("Retrieve SubNets") subnets, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ DryRun: nil, @@ -51,9 +51,9 @@ func (c client) GetSubNets(vpc *string) ([]*string, error) { return nil, err } - ids := []*string{} + ids := []string{} for _, subnet := range subnets.Subnets { - ids = append(ids, subnet.SubnetId) + ids = append(ids, *subnet.SubnetId) } return ids, nil } @@ -91,7 +91,6 @@ func (c client) CreateSecurityGroup(project *compose.Project, vpc *string) (*str return securityGroup.GroupId, nil } - func (c *client) ExposePort(securityGroup *string, port types.ServicePortConfig) error { logrus.Debugf("Authorize ingress port %d/%s\n", port.Published, port.Protocol) _, err := c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ @@ -110,4 +109,4 @@ func (c *client) ExposePort(securityGroup *string, port types.ServicePortConfig) }, }) return err -} \ No newline at end of file +} diff --git a/ecs/pkg/amazon/roles.go b/ecs/pkg/amazon/roles.go index bfa5026a..7e4594af 100644 --- a/ecs/pkg/amazon/roles.go +++ b/ecs/pkg/amazon/roles.go @@ -2,6 +2,7 @@ package amazon import ( "fmt" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" "github.com/compose-spec/compose-go/types" @@ -13,7 +14,7 @@ const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTa var defaultTaskExecutionRole *string // GetEcsTaskExecutionRole retrieve the role ARN to apply for task execution -func (c client) GetEcsTaskExecutionRole(spec *types.ServiceConfig) (*string, error) { +func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (*string, error) { if arn, ok := spec.Extras["x-ecs-TaskExecutionRole"]; ok { s := arn.(string) return &s, nil diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 0dbc35c9..5f9b8750 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -2,128 +2,54 @@ package amazon import ( "fmt" + "os" + "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ecs" - "github.com/compose-spec/compose-go/types" + "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/pkg/convert" - "github.com/sirupsen/logrus" ) func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) error { - type mapping struct { - service *types.ServiceConfig - task *ecs.RegisterTaskDefinitionInput - } - mappings := []mapping{} - for _, service := range project.Services { - task, err := convert.Convert(project, service) - if err != nil { - return err - } - mappings = append(mappings, mapping{ - service: &service, - task: task, - }) - } - - vpc, err := c.GetDefaultVPC() - if err != nil { - return err - } - subnets, err := c.GetSubNets(vpc) + template, err := c.Convert(project, loadBalancerArn) if err != nil { return err } - securityGroup, err := c.CreateSecurityGroup(project, vpc) + json, err := template.JSON() if err != nil { return err } - if loadBalancerArn == nil { - loadBalancerArn, err = c.CreateLoadBalancer(project, subnets) - if err != nil { - return err - } - } - - logGroup, err := c.GetOrCreateLogGroup(project) + _, err = c.CF.ValidateTemplate(&cloudformation.ValidateTemplateInput{ + TemplateBody: aws.String(string(json)), + }) if err != nil { return err } - for _, mapping := range mappings { - ingress := []*ecs.LoadBalancer{} - for _, port := range mapping.service.Ports { - name := fmt.Sprintf("%s-%s-%d-%s", project.Name, mapping.service.Name, port.Target, port.Protocol) - targetgroup, err := c.CreateTargetGroup(name, vpc, port) - if err != nil { - return err - } - ingress = append(ingress, &ecs.LoadBalancer{ - ContainerName: aws.String(mapping.service.Name), - ContainerPort: aws.Int64(int64(port.Target)), - TargetGroupArn: targetgroup, - }) + _, err = c.CF.CreateStack(&cloudformation.CreateStackInput{ + OnFailure: aws.String("DELETE"), + StackName: aws.String(project.Name), + TemplateBody: aws.String(string(json)), + TimeoutInMinutes: aws.Int64(10), + }) + if err != nil { + return err + } - err = c.CreateListener(port, loadBalancerArn, targetgroup) - if err != nil { - return err - } - } - - _, err = c.CreateService(project, mapping.service, mapping.task, securityGroup, subnets, logGroup, ingress) - if err != nil { - return err + events, err := c.CF.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{ + StackName: aws.String(project.Name), + }) + if err != nil { + return err + } + for _, event := range events.StackEvents { + fmt.Printf("%s %s\n", *event.LogicalResourceId, *event.ResourceStatus) + if *event.ResourceStatus == "CREATE_FAILED" { + fmt.Fprintln(os.Stderr, event.ResourceStatusReason) } } + + // TODO monitor progress return nil } - -func (c *client) CreateService(project *compose.Project, service *types.ServiceConfig, task *ecs.RegisterTaskDefinitionInput, securityGroup *string, subnets []*string, logGroup *string, ingress []*ecs.LoadBalancer) (*string, error) { - role, err := c.GetEcsTaskExecutionRole(service) - if err != nil { - return nil, err - } - - task.ExecutionRoleArn = role - - for _, def := range task.ContainerDefinitions { - def.LogConfiguration.Options["awslogs-group"] = logGroup - def.LogConfiguration.Options["awslogs-stream-prefix"] = aws.String(service.Name) - def.LogConfiguration.Options["awslogs-region"] = aws.String(c.Region) - } - - arn, err := c.RegisterTaskDefinition(task) - if err != nil { - return nil, err - } - - logrus.Debug("Create Service") - created, err := c.ECS.CreateService(&ecs.CreateServiceInput{ - Cluster: aws.String(c.Cluster), - DesiredCount: aws.Int64(1), // FIXME get from deploy options - LaunchType: aws.String(ecs.LaunchTypeFargate), //FIXME use service.Isolation tro select EC2 vs Fargate - NetworkConfiguration: &ecs.NetworkConfiguration{ - AwsvpcConfiguration: &ecs.AwsVpcConfiguration{ - AssignPublicIp: aws.String(ecs.AssignPublicIpEnabled), - SecurityGroups: []*string{securityGroup}, - Subnets: subnets, - }, - }, - ServiceName: aws.String(service.Name), - SchedulingStrategy: aws.String(ecs.SchedulingStrategyReplica), - TaskDefinition: arn, - LoadBalancers: ingress, - }) - - for _, port := range service.Ports { - err = c.ExposePort(securityGroup, port) - if err != nil { - return nil, err - } - } - - return created.Service.ServiceArn, err -} diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index ceebacb8..50f62ccf 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -1,6 +1,9 @@ package compose +import "github.com/awslabs/goformation/v4/cloudformation" + type API interface { + Convert(project *Project, loadBalancerArn *string) (*cloudformation.Template, error) ComposeUp(project *Project, loadBalancerArn *string) error ComposeDown(project *Project, keepLoadBalancer bool) error -} \ No newline at end of file +} diff --git a/ecs/pkg/compose/opts.go b/ecs/pkg/compose/opts.go index 9429fb01..390ccaa6 100644 --- a/ecs/pkg/compose/opts.go +++ b/ecs/pkg/compose/opts.go @@ -15,7 +15,6 @@ func (o *ProjectOptions) AddFlags(flags *pflag.FlagSet) { flags.StringVarP(&o.name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") } - type ProjectFunc func(project *Project, args []string) error // WithProject wrap a ProjectFunc into a cobra command diff --git a/ecs/pkg/compose/project.go b/ecs/pkg/compose/project.go index c2c6c007..6f2ef98e 100644 --- a/ecs/pkg/compose/project.go +++ b/ecs/pkg/compose/project.go @@ -2,14 +2,15 @@ package compose import ( "fmt" - "github.com/compose-spec/compose-go/loader" - "github.com/compose-spec/compose-go/types" - "github.com/sirupsen/logrus" "io/ioutil" "os" "path/filepath" "regexp" "strings" + + "github.com/compose-spec/compose-go/loader" + "github.com/compose-spec/compose-go/types" + "github.com/sirupsen/logrus" ) type Project struct { @@ -32,7 +33,6 @@ func NewProject(config types.ConfigDetails, name string) (*Project, error) { return &p, nil } - // projectFromOptions load a compose project based on command line options func projectFromOptions(options *ProjectOptions) (*Project, error) { configPath, err := getConfigPathFromOptions(options) diff --git a/ecs/pkg/compose/project_test.go b/ecs/pkg/compose/project_test.go index d5f77140..0b44fb33 100644 --- a/ecs/pkg/compose/project_test.go +++ b/ecs/pkg/compose/project_test.go @@ -1,9 +1,10 @@ package compose import ( - "gotest.tools/v3/assert" "os" "testing" + + "gotest.tools/v3/assert" ) func Test_project_name(t *testing.T) { diff --git a/ecs/pkg/convert/convert.go b/ecs/pkg/convert/convert.go index 502f5089..33e95c73 100644 --- a/ecs/pkg/convert/convert.go +++ b/ecs/pkg/convert/convert.go @@ -1,118 +1,87 @@ package convert import ( - "github.com/docker/ecs-plugin/pkg/compose" + "strings" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ecs" + ecsapi "github.com/aws/aws-sdk-go/service/ecs" + "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/compose-spec/compose-go/types" "github.com/docker/cli/opts" + "github.com/docker/ecs-plugin/pkg/compose" ) -func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.RegisterTaskDefinitionInput, error) { +func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { _, err := toCPULimits(service) if err != nil { return nil, err } - foo := int64(256) - logDriver := "awslogs" // FIXME could be set by service.Logging, especially to enable use of firelens - return &ecs.RegisterTaskDefinitionInput{ - ContainerDefinitions: []*ecs.ContainerDefinition{ + return &ecs.TaskDefinition{ + ContainerDefinitions: []ecs.TaskDefinition_ContainerDefinition{ // Here we can declare sidecars and init-containers using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_dependson { - Command: toStringPtrSlice(service.Command), - Cpu: &foo, - DependsOn: nil, - DisableNetworking: toBoolPtr(service.NetworkMode == "none"), - DnsSearchDomains: toStringPtrSlice(service.DNSSearch), - DnsServers: toStringPtrSlice(service.DNS), - DockerLabels: nil, - DockerSecurityOptions: toStringPtrSlice(service.SecurityOpt), - EntryPoint: toStringPtrSlice(service.Entrypoint), - Environment: toKeyValuePairPtr(service.Environment), - Essential: toBoolPtr(true), - ExtraHosts: toHostEntryPtr(service.ExtraHosts), - FirelensConfiguration: nil, - HealthCheck: toHealthCheck(service.HealthCheck), - Hostname: toStringPtr(service.Hostname), - Image: toStringPtr(service.Image), - Interactive: nil, - Links: nil, - LinuxParameters: toLinuxParameters(service), - LogConfiguration: &ecs.LogConfiguration{ - LogDriver: &logDriver, - Options: map[string]*string{}, - SecretOptions: nil, - }, + Command: service.Command, + Cpu: 256, + DisableNetworking: service.NetworkMode == "none", + DnsSearchDomains: service.DNSSearch, + DnsServers: service.DNS, + DockerLabels: nil, + DockerSecurityOptions: service.SecurityOpt, + EntryPoint: service.Entrypoint, + Environment: toKeyValuePair(service.Environment), + Essential: true, + ExtraHosts: toHostEntryPtr(service.ExtraHosts), + FirelensConfiguration: nil, + HealthCheck: toHealthCheck(service.HealthCheck), + Hostname: service.Hostname, + Image: service.Image, + Interactive: false, + Links: nil, + LinuxParameters: toLinuxParameters(service), Memory: toMemoryLimits(service.Deploy), MemoryReservation: toMemoryReservation(service.Deploy), MountPoints: nil, - Name: toStringPtr(service.Name), + Name: service.Name, PortMappings: toPortMappings(service.Ports), - Privileged: toBoolPtr(service.Privileged), - PseudoTerminal: toBoolPtr(service.Tty), - ReadonlyRootFilesystem: toBoolPtr(service.ReadOnly), + Privileged: service.Privileged, + PseudoTerminal: service.Tty, + ReadonlyRootFilesystem: service.ReadOnly, RepositoryCredentials: nil, ResourceRequirements: nil, Secrets: nil, - StartTimeout: nil, - StopTimeout: durationToInt64Ptr(service.StopGracePeriod), + StartTimeout: 0, + StopTimeout: durationToInt(service.StopGracePeriod), SystemControls: nil, Ulimits: toUlimits(service.Ulimits), - User: toStringPtr(service.User), + User: service.User, VolumesFrom: nil, - WorkingDirectory: toStringPtr(service.WorkingDir), + WorkingDirectory: service.WorkingDir, }, }, Cpu: toCPU(service), - ExecutionRoleArn: nil, - Family: toStringPtr(project.Name), - IpcMode: toStringPtr(service.Ipc), + Family: project.Name, + IpcMode: service.Ipc, Memory: toMemory(service), - NetworkMode: toStringPtr("awsvpc"), // FIXME could be set by service.NetworkMode, Fargate only supports network mode ‘awsvpc’. - PidMode: toStringPtr(service.Pid), + NetworkMode: ecsapi.NetworkModeAwsvpc, // FIXME could be set by service.NetworkMode, Fargate only supports network mode ‘awsvpc’. + PidMode: service.Pid, PlacementConstraints: toPlacementConstraints(service.Deploy), ProxyConfiguration: nil, - RequiresCompatibilities: toRequiresCompatibilities(ecs.LaunchTypeFargate), + RequiresCompatibilities: []string{ecsapi.LaunchTypeFargate}, Tags: nil, - Volumes: []*ecs.Volume{ - { - /* ONLY supported when using EC2 launch type - DockerVolumeConfiguration: { - Autoprovision: nil, - Driver: nil, - DriverOpts: nil, - Labels: nil, - Scope: nil, - }, */ - /* Beta and ONLY supported when using EC2 launch type - EfsVolumeConfiguration: { - FileSystemId: nil, - RootDirectory: nil, - }, */ - /* Bind mount host volume - Host: { - SourcePath: - }, */ - Name: aws.String("MyVolume"), - }, - }, + Volumes: []ecs.TaskDefinition_Volume{}, }, nil } -func toCPU(service types.ServiceConfig) *string { +func toCPU(service types.ServiceConfig) string { // FIXME based on service's memory/cpu requirements, select the adequate Fargate CPU - v := "256" - return &v + return "256" } -func toMemory(service types.ServiceConfig) *string { +func toMemory(service types.ServiceConfig) string { // FIXME based on service's memory/cpu requirements, select the adequate Fargate CPU - v := "512" - return &v + return "512" } func toCPULimits(service types.ServiceConfig) (*int64, error) { @@ -153,45 +122,45 @@ func hasMemoryOrMemoryReservation(service types.ServiceConfig) bool { return false } -func toPlacementConstraints(deploy *types.DeployConfig) []*ecs.TaskDefinitionPlacementConstraint { +func toPlacementConstraints(deploy *types.DeployConfig) []ecs.TaskDefinition_TaskDefinitionPlacementConstraint { if deploy == nil || deploy.Placement.Constraints == nil || len(deploy.Placement.Constraints) == 0 { return nil } - pl := []*ecs.TaskDefinitionPlacementConstraint{} + pl := []ecs.TaskDefinition_TaskDefinitionPlacementConstraint{} for _, c := range deploy.Placement.Constraints { - pl = append(pl, &ecs.TaskDefinitionPlacementConstraint{ - Expression: toStringPtr(c), - Type: nil, + pl = append(pl, ecs.TaskDefinition_TaskDefinitionPlacementConstraint{ + Expression: c, + Type: "", }) } return pl } -func toPortMappings(ports []types.ServicePortConfig) []*ecs.PortMapping { +func toPortMappings(ports []types.ServicePortConfig) []ecs.TaskDefinition_PortMapping { if len(ports) == 0 { return nil } - m := []*ecs.PortMapping{} + m := []ecs.TaskDefinition_PortMapping{} for _, p := range ports { - m = append(m, &ecs.PortMapping{ - ContainerPort: uint32Toint64Ptr(p.Target), - HostPort: uint32Toint64Ptr(p.Published), - Protocol: toStringPtr(p.Protocol), + m = append(m, ecs.TaskDefinition_PortMapping{ + ContainerPort: int(p.Target), + HostPort: int(p.Published), + Protocol: p.Protocol, }) } return m } -func toUlimits(ulimits map[string]*types.UlimitsConfig) []*ecs.Ulimit { +func toUlimits(ulimits map[string]*types.UlimitsConfig) []ecs.TaskDefinition_Ulimit { if len(ulimits) == 0 { return nil } - u := []*ecs.Ulimit{} + u := []ecs.TaskDefinition_Ulimit{} for k, v := range ulimits { - u = append(u, &ecs.Ulimit{ - Name: toStringPtr(k), - SoftLimit: intToInt64Ptr(v.Soft), - HardLimit: intToInt64Ptr(v.Hard), + u = append(u, ecs.TaskDefinition_Ulimit{ + Name: k, + SoftLimit: v.Soft, + HardLimit: v.Hard, }) } return u @@ -209,79 +178,82 @@ func intToInt64Ptr(i int) *int64 { const Mb = 1024 * 1024 -func toMemoryLimits(deploy *types.DeployConfig) *int64 { +func toMemoryLimits(deploy *types.DeployConfig) int { if deploy == nil { - return nil + return 0 } res := deploy.Resources.Limits if res == nil { - return nil + return 0 } - v := int64(res.MemoryBytes) / Mb - return &v + v := int(res.MemoryBytes) / Mb + return v } -func toMemoryReservation(deploy *types.DeployConfig) *int64 { +func toMemoryReservation(deploy *types.DeployConfig) int { if deploy == nil { - return nil + return 0 } res := deploy.Resources.Reservations if res == nil { - return nil + return 0 } - v := int64(res.MemoryBytes) / Mb - return &v + v := int(res.MemoryBytes) / Mb + return v } -func toLinuxParameters(service types.ServiceConfig) *ecs.LinuxParameters { - return &ecs.LinuxParameters{ +func toLinuxParameters(service types.ServiceConfig) *ecs.TaskDefinition_LinuxParameters { + return &ecs.TaskDefinition_LinuxParameters{ Capabilities: toKernelCapabilities(service.CapAdd, service.CapDrop), Devices: nil, - InitProcessEnabled: service.Init, - MaxSwap: nil, + InitProcessEnabled: service.Init != nil && *service.Init, + MaxSwap: 0, // FIXME SharedMemorySize: service.ShmSize, - Swappiness: nil, + Swappiness: 0, Tmpfs: toTmpfs(service.Tmpfs), } } -func toTmpfs(tmpfs types.StringList) []*ecs.Tmpfs { +func toTmpfs(tmpfs types.StringList) []ecs.TaskDefinition_Tmpfs { if tmpfs == nil || len(tmpfs) == 0 { return nil } - o := []*ecs.Tmpfs{} - for _, t := range tmpfs { - path := t - o = append(o, &ecs.Tmpfs{ - ContainerPath: &path, + o := []ecs.TaskDefinition_Tmpfs{} + for _, path := range tmpfs { + o = append(o, ecs.TaskDefinition_Tmpfs{ + ContainerPath: path, MountOptions: nil, - Size: nil, + Size: 0, }) } return o } -func toKernelCapabilities(add []string, drop []string) *ecs.KernelCapabilities { +func toKernelCapabilities(add []string, drop []string) *ecs.TaskDefinition_KernelCapabilities { if len(add) == 0 && len(drop) == 0 { return nil } - return &ecs.KernelCapabilities{ - Add: toStringPtrSlice(add), - Drop: toStringPtrSlice(drop), + return &ecs.TaskDefinition_KernelCapabilities{ + Add: add, + Drop: drop, } } -func toHealthCheck(check *types.HealthCheckConfig) *ecs.HealthCheck { +func toHealthCheck(check *types.HealthCheckConfig) *ecs.TaskDefinition_HealthCheck { if check == nil { return nil } - return &ecs.HealthCheck{ - Command: toStringPtrSlice(check.Test), - Interval: durationToInt64Ptr(check.Interval), - Retries: uint64ToInt64Ptr(check.Retries), - StartPeriod: durationToInt64Ptr(check.StartPeriod), - Timeout: durationToInt64Ptr(check.Timeout), + retries := 0 + if check.Retries != nil { + retries = int(*check.Retries) + } + return &ecs.TaskDefinition_HealthCheck{ + Command: check.Test, + Interval: durationToInt(check.Interval), + Retries: retries, + StartPeriod: durationToInt(check.StartPeriod), + Timeout: durationToInt(check.Timeout), } } @@ -293,66 +265,44 @@ func uint64ToInt64Ptr(i *uint64) *int64 { return &v } -func durationToInt64Ptr(interval *types.Duration) *int64 { +func durationToInt(interval *types.Duration) int { if interval == nil { - return nil + return 0 } - v := int64(time.Duration(*interval).Seconds()) - return &v + v := int(time.Duration(*interval).Seconds()) + return v } -func toHostEntryPtr(hosts types.HostsList) []*ecs.HostEntry { +func toHostEntryPtr(hosts types.HostsList) []ecs.TaskDefinition_HostEntry { if hosts == nil || len(hosts) == 0 { return nil } - e := []*ecs.HostEntry{} + e := []ecs.TaskDefinition_HostEntry{} for _, h := range hosts { - host := h - e = append(e, &ecs.HostEntry{ - Hostname: &host, + parts := strings.SplitN(h, ":", 2) // FIXME this should be handled by compose-go + e = append(e, ecs.TaskDefinition_HostEntry{ + Hostname: parts[0], + IpAddress: parts[1], }) } return e } -func toKeyValuePairPtr(environment types.MappingWithEquals) []*ecs.KeyValuePair { +func toKeyValuePair(environment types.MappingWithEquals) []ecs.TaskDefinition_KeyValuePair { if environment == nil || len(environment) == 0 { return nil } - pairs := []*ecs.KeyValuePair{} + pairs := []ecs.TaskDefinition_KeyValuePair{} for k, v := range environment { name := k - value := v - pairs = append(pairs, &ecs.KeyValuePair{ - Name: &name, + var value string + if v != nil { + value = *v + } + pairs = append(pairs, ecs.TaskDefinition_KeyValuePair{ + Name: name, Value: value, }) } return pairs } - -func toStringPtr(s string) *string { - if s == "" { - return nil - } - return &s -} - -func toStringPtrSlice(s []string) []*string { - if len(s) == 0 { - return nil - } - v := []*string{} - for _, x := range s { - value := x - v = append(v, &value) - } - return v -} - -func toBoolPtr(b bool) *bool { - if !b { - return nil - } - return &b -} \ No newline at end of file From 0972776e6d469113570c84a2fa1f4c4c35919710 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 21 Apr 2020 14:48:31 +0200 Subject: [PATCH 014/205] Ingress description to include service being exposed Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 0b3b0bc4..1865aa82 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -30,7 +30,7 @@ func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*clo for _, port := range service.Ports { ingresses = append(ingresses, ec2.SecurityGroup_Ingress{ CidrIp: "0.0.0.0/0", - Description: fmt.Sprintf("%d/%s", port.Target, port.Protocol), + Description: fmt.Sprintf("%s:%d/%s", service.Name, port.Target, port.Protocol), FromPort: int(port.Target), IpProtocol: strings.ToUpper(port.Protocol), ToPort: int(port.Target), From 30fd37b6cabb96b99e5a2754342494e7f7ab25cf Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 22 Apr 2020 15:06:01 +0200 Subject: [PATCH 015/205] ecs cluster create Signed-off-by: aiordache --- ecs/pkg/amazon/down.go | 2 +- ecs/pkg/amazon/ecs.go | 37 +++++++++++++++++++++++++++++++++++++ ecs/pkg/amazon/up.go | 7 +++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index ac264832..44f49351 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -12,7 +12,7 @@ func (c *client) ComposeDown(project *compose.Project, keepLoadBalancer bool) er if err != nil { return err } - + c.DeleteCluster() // TODO monitor progress return nil } diff --git a/ecs/pkg/amazon/ecs.go b/ecs/pkg/amazon/ecs.go index 423980f3..232d1d60 100644 --- a/ecs/pkg/amazon/ecs.go +++ b/ecs/pkg/amazon/ecs.go @@ -1,6 +1,9 @@ package amazon import ( + "errors" + "strings" + "github.com/aws/aws-sdk-go/service/ecs" "github.com/sirupsen/logrus" ) @@ -13,3 +16,37 @@ func (c client) RegisterTaskDefinition(task *ecs.RegisterTaskDefinitionInput) (* } return def.TaskDefinition.TaskDefinitionArn, err } + +func (c client) CreateCluster() (*string, error) { + logrus.Debug("Create cluster ", c.Cluster) + response, err := c.ECS.CreateCluster(&ecs.CreateClusterInput{ClusterName: &c.Cluster}) + if err != nil { + return nil, err + } + return response.Cluster.Status, nil +} + +func (c client) DeleteCluster() error { + logrus.Debug("Delete cluster ", c.Cluster) + response, err := c.ECS.DeleteCluster(&ecs.DeleteClusterInput{Cluster: &c.Cluster}) + if err != nil { + return err + } + if *response.Cluster.Status == "INACTIVE" { + return nil + } + return errors.New("Failed to delete cluster, status: " + *response.Cluster.Status) +} + +func (c client) ClusterExists() (bool, error) { + logrus.Debug("Check if cluster was already created: ", c.Cluster) + clusters, err := c.ECS.ListClusters(nil) + if err != nil { + return false, err + } + found := false + for _, arn := range clusters.ClusterArns { + found = found || strings.HasSuffix(*arn, "/"+c.Cluster) + } + return found, nil +} diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 5f9b8750..baa11793 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -10,6 +10,13 @@ import ( ) func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) error { + ok, err := c.ClusterExists() + if err != nil { + return err + } + if !ok { + c.CreateCluster() + } template, err := c.Convert(project, loadBalancerArn) if err != nil { return err From 87f053d7104515848d706be5951dc8a17edfcfb5 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 22 Apr 2020 15:37:39 +0200 Subject: [PATCH 016/205] Detect stack already exists This will later be used to switch to ChangeSet logic Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 1 - ecs/pkg/amazon/up.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 1865aa82..dd304ab4 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -14,7 +14,6 @@ import ( func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*cloudformation.Template, error) { template := cloudformation.NewTemplate() - vpc, err := c.GetDefaultVPC() if err != nil { return nil, err diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 5f9b8750..efb78f2b 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -10,6 +10,16 @@ import ( ) func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) error { + stacks, err := c.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ + StackName: aws.String(project.Name), + }) + if err != nil { + return err + } + if len(stacks.Stacks) > 0 { + return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") + } + template, err := c.Convert(project, loadBalancerArn) if err != nil { return err From 5110cb6b851e109012b243a31f012a07653e5470 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 22 Apr 2020 15:39:02 +0200 Subject: [PATCH 017/205] Basic architecture documentation Signed-off-by: Nicolas De Loof --- ecs/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ecs/README.md b/ecs/README.md index 128f6e0c..9ec4ab69 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -1 +1,31 @@ # Docker CLI plugin for Amazon ECS + +## Architecture + +ECS plugin is a [Docker CLI plugin](https://docs.docker.com/engine/extend/cli_plugins/) +root command `ecs` require aws profile to get API credentials from `~/.aws/credentials` +as well as AWS region - those will later be stored in a docker context + +A `compose.yaml` is parsed and converted into a [CloudFormation](https://aws.amazon.com/cloudformation/) +template, which will create all resources in dependent order and cleanup on +`down` command or deployment failure. + +``` + +-----------------------------+ + | compose.yaml file | + +-----------------------------+ +- Load + +-----------------------------+ + | compose-go Model | + +-----------------------------+ +- Convert + +-----------------------------+ + | CloudFormation Template | + +-----------------------------+ +- Apply + +---------+ +------------+ + | AWS API | or | stack file | + +---------+ +------------+ +``` + +(if this sounds familiar, see [Kompose](https://github.com/kubernetes/kompose/blob/master/docs/architecture.md)) \ No newline at end of file From 55f2908c16dd503e1a72a750ccb9268adea18f11 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 22 Apr 2020 16:42:52 +0200 Subject: [PATCH 018/205] wait for stack removal on cluster delete Signed-off-by: aiordache --- ecs/go.mod | 4 +++- ecs/go.sum | 48 ++++++++++++++++++++++++++++++++++++++++++ ecs/pkg/amazon/down.go | 14 ++++++++++-- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/ecs/go.mod b/ecs/go.mod index a8f444c5..88254c27 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -38,7 +38,7 @@ require ( github.com/opencontainers/image-spec v1.0.1 // indirect github.com/sirupsen/logrus v1.5.0 github.com/spf13/cobra v0.0.5 - github.com/spf13/pflag v1.0.3 + github.com/spf13/pflag v1.0.5 github.com/theupdateframework/notary v0.6.1 // indirect github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect @@ -46,7 +46,9 @@ require ( gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect + gotest.tools v2.2.0+incompatible gotest.tools/v3 v3.0.2 + k8s.io/apimachinery v0.18.2 vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect ) diff --git a/ecs/go.sum b/ecs/go.sum index dc5a0393..31441e0d 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -11,6 +11,9 @@ github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6tr github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= @@ -91,17 +94,27 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= @@ -118,7 +131,9 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -130,6 +145,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= @@ -159,6 +178,7 @@ github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVz github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= @@ -182,6 +202,7 @@ github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -204,12 +225,17 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.2.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -268,9 +294,12 @@ github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -325,6 +354,7 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -337,6 +367,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYR golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -345,6 +376,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -360,6 +392,9 @@ golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdO golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -368,6 +403,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqG golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -402,6 +438,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y2O4MU= gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -414,6 +451,17 @@ gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/apimachinery v0.18.2 h1:44CmtbmkzVDAhCpRVSiP2R5PPrC2RtlIv/MoB8xpdRA= +k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index 44f49351..4976aef1 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -1,7 +1,10 @@ package amazon import ( + "fmt" + "github.com/aws/aws-sdk-go/service/cloudformation" + cf "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -12,7 +15,14 @@ func (c *client) ComposeDown(project *compose.Project, keepLoadBalancer bool) er if err != nil { return err } - c.DeleteCluster() - // TODO monitor progress + fmt.Printf("Delete stack ") + if err = c.CF.WaitUntilStackDeleteComplete(&cf.DescribeStacksInput{StackName: &project.Name}); err != nil { + return err + } + fmt.Printf("... done.\nDelete cluster %s", c.Cluster) + if err = c.DeleteCluster(); err != nil { + return err + } + fmt.Printf("... done. \n") return nil } From 3c9905c4747560dd4dd4e7b4d4f34eade8c6e007 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 23 Apr 2020 09:45:50 +0200 Subject: [PATCH 019/205] tidy up go mod Signed-off-by: aiordache --- ecs/go.mod | 4 ++-- ecs/go.sum | 44 +------------------------------------------- 2 files changed, 3 insertions(+), 45 deletions(-) diff --git a/ecs/go.mod b/ecs/go.mod index 88254c27..0bc80e99 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -35,20 +35,20 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/miekg/pkcs11 v1.0.3 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/onsi/ginkgo v1.11.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/sirupsen/logrus v1.5.0 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/theupdateframework/notary v0.6.1 // indirect github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect + golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/grpc v1.27.0 // indirect gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect - gotest.tools v2.2.0+incompatible gotest.tools/v3 v3.0.2 - k8s.io/apimachinery v0.18.2 vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect ) diff --git a/ecs/go.sum b/ecs/go.sum index 31441e0d..ee07c608 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -11,9 +11,6 @@ github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6tr github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= @@ -94,27 +91,17 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= @@ -131,9 +118,7 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -145,10 +130,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= @@ -178,7 +159,6 @@ github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVz github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= @@ -202,7 +182,6 @@ github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -225,17 +204,14 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.2.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -294,7 +270,6 @@ github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -354,7 +329,6 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -367,7 +341,6 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYR golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -376,7 +349,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -394,7 +366,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -403,7 +374,6 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqG golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -438,7 +408,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y2O4MU= gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -451,17 +420,6 @@ gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/apimachinery v0.18.2 h1:44CmtbmkzVDAhCpRVSiP2R5PPrC2RtlIv/MoB8xpdRA= -k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= From 48096eeed8b2e0fd25266f5d77a9565073c87e8e Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 23 Apr 2020 11:19:59 +0200 Subject: [PATCH 020/205] DescribeStacks fail with error if stack does not exists Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/up.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index efb78f2b..4cd52115 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -10,13 +10,11 @@ import ( ) func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) error { - stacks, err := c.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ + _, err := c.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ StackName: aws.String(project.Name), }) - if err != nil { - return err - } - if len(stacks.Stacks) > 0 { + if err == nil { + // FIXME no ErrNotFound err type here return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") } From 3d7e062215924e6e19a2af5927aab14ede2a9fbd Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 23 Apr 2020 14:50:18 +0200 Subject: [PATCH 021/205] add delete-cluster flag on down cmd Signed-off-by: aiordache --- ecs/cmd/main/main.go | 4 +++- ecs/pkg/amazon/down.go | 10 ++++++++-- ecs/pkg/compose/api.go | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index cde9be3b..ab365bd4 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -133,6 +133,7 @@ func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) type downOptions struct { KeepLoadBalancer bool + DeleteCluster bool } func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { @@ -144,9 +145,10 @@ func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOption if err != nil { return err } - return client.ComposeDown(project, opts.KeepLoadBalancer) + return client.ComposeDown(project, opts.KeepLoadBalancer, opts.DeleteCluster) }), } cmd.Flags().BoolVar(&opts.KeepLoadBalancer, "keep-load-balancer", false, "Keep Load Balancer for further use") + cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") return cmd } diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index 4976aef1..a7b44892 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -8,7 +8,7 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) -func (c *client) ComposeDown(project *compose.Project, keepLoadBalancer bool) error { +func (c *client) ComposeDown(project *compose.Project, keepLoadBalancer, deleteCluster bool) error { _, err := c.CF.DeleteStack(&cloudformation.DeleteStackInput{ StackName: &project.Name, }) @@ -19,7 +19,13 @@ func (c *client) ComposeDown(project *compose.Project, keepLoadBalancer bool) er if err = c.CF.WaitUntilStackDeleteComplete(&cf.DescribeStacksInput{StackName: &project.Name}); err != nil { return err } - fmt.Printf("... done.\nDelete cluster %s", c.Cluster) + fmt.Printf("... done.\n") + + if !deleteCluster { + return nil + } + + fmt.Printf("Delete cluster %s", c.Cluster) if err = c.DeleteCluster(); err != nil { return err } diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 50f62ccf..4218de76 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -5,5 +5,5 @@ import "github.com/awslabs/goformation/v4/cloudformation" type API interface { Convert(project *Project, loadBalancerArn *string) (*cloudformation.Template, error) ComposeUp(project *Project, loadBalancerArn *string) error - ComposeDown(project *Project, keepLoadBalancer bool) error + ComposeDown(project *Project, keepLoadBalancer, deleteCluster bool) error } From ea6d35a9274a2ee24ca698f2956ba89775825661 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 23 Apr 2020 16:19:47 +0200 Subject: [PATCH 022/205] Fix minor issue after merge conflit resolution Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/up.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 8b6c6da6..2dbfd306 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -17,7 +17,7 @@ func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) er if !ok { c.CreateCluster() } - _, err := c.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ + _, err = c.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ StackName: aws.String(project.Name), }) if err == nil { From f8bf0078aa165460ceac3181c397486fbd92f15f Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 24 Apr 2020 10:13:38 +0200 Subject: [PATCH 023/205] Use DescribeCluster as ListCluster is a Paginated API Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/ecs.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ecs/pkg/amazon/ecs.go b/ecs/pkg/amazon/ecs.go index 232d1d60..03ca92f4 100644 --- a/ecs/pkg/amazon/ecs.go +++ b/ecs/pkg/amazon/ecs.go @@ -2,8 +2,7 @@ package amazon import ( "errors" - "strings" - + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" "github.com/sirupsen/logrus" ) @@ -40,13 +39,11 @@ func (c client) DeleteCluster() error { func (c client) ClusterExists() (bool, error) { logrus.Debug("Check if cluster was already created: ", c.Cluster) - clusters, err := c.ECS.ListClusters(nil) + clusters, err := c.ECS.DescribeClusters(&ecs.DescribeClustersInput{ + Clusters: []*string{aws.String(c.Cluster)}, + }) if err != nil { return false, err } - found := false - for _, arn := range clusters.ClusterArns { - found = found || strings.HasSuffix(*arn, "/"+c.Cluster) - } - return found, nil + return len(clusters.Clusters) > 0, nil } From d612a4ab894c43178713fa1ef6a75ec10091e11e Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 24 Apr 2020 17:25:29 +0200 Subject: [PATCH 024/205] Project name parameter as alternative to compose file on down Signed-off-by: aiordache --- ecs/cmd/main/main.go | 20 +++++++++++++++++--- ecs/pkg/amazon/down.go | 7 +++---- ecs/pkg/compose/api.go | 2 +- ecs/pkg/compose/opts.go | 2 +- ecs/pkg/compose/project.go | 2 +- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index ab365bd4..7b3d1d6a 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -140,13 +140,27 @@ func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOption opts := downOptions{} cmd := &cobra.Command{ Use: "down", - RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) if err != nil { return err } - return client.ComposeDown(project, opts.KeepLoadBalancer, opts.DeleteCluster) - }), + if len(args) == 0 { + project, err := compose.ProjectFromOptions(projectOpts) + if err != nil { + return err + } + return client.ComposeDown(&project.Name, opts.KeepLoadBalancer, opts.DeleteCluster) + } + // project names passed as parameters + for _, name := range args { + err := client.ComposeDown(&name, opts.KeepLoadBalancer, opts.DeleteCluster) + if err != nil { + return err + } + } + return nil + }, } cmd.Flags().BoolVar(&opts.KeepLoadBalancer, "keep-load-balancer", false, "Keep Load Balancer for further use") cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index a7b44892..4dfdff4a 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -5,18 +5,17 @@ import ( "github.com/aws/aws-sdk-go/service/cloudformation" cf "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/docker/ecs-plugin/pkg/compose" ) -func (c *client) ComposeDown(project *compose.Project, keepLoadBalancer, deleteCluster bool) error { +func (c *client) ComposeDown(projectName *string, keepLoadBalancer, deleteCluster bool) error { _, err := c.CF.DeleteStack(&cloudformation.DeleteStackInput{ - StackName: &project.Name, + StackName: projectName, }) if err != nil { return err } fmt.Printf("Delete stack ") - if err = c.CF.WaitUntilStackDeleteComplete(&cf.DescribeStacksInput{StackName: &project.Name}); err != nil { + if err = c.CF.WaitUntilStackDeleteComplete(&cf.DescribeStacksInput{StackName: projectName}); err != nil { return err } fmt.Printf("... done.\n") diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 4218de76..b3075082 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -5,5 +5,5 @@ import "github.com/awslabs/goformation/v4/cloudformation" type API interface { Convert(project *Project, loadBalancerArn *string) (*cloudformation.Template, error) ComposeUp(project *Project, loadBalancerArn *string) error - ComposeDown(project *Project, keepLoadBalancer, deleteCluster bool) error + ComposeDown(projectName *string, keepLoadBalancer, deleteCluster bool) error } diff --git a/ecs/pkg/compose/opts.go b/ecs/pkg/compose/opts.go index 390ccaa6..9e9bdeee 100644 --- a/ecs/pkg/compose/opts.go +++ b/ecs/pkg/compose/opts.go @@ -20,7 +20,7 @@ type ProjectFunc func(project *Project, args []string) error // WithProject wrap a ProjectFunc into a cobra command func WithProject(options *ProjectOptions, f ProjectFunc) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { - project, err := projectFromOptions(options) + project, err := ProjectFromOptions(options) if err != nil { return err } diff --git a/ecs/pkg/compose/project.go b/ecs/pkg/compose/project.go index 6f2ef98e..e17e2252 100644 --- a/ecs/pkg/compose/project.go +++ b/ecs/pkg/compose/project.go @@ -34,7 +34,7 @@ func NewProject(config types.ConfigDetails, name string) (*Project, error) { } // projectFromOptions load a compose project based on command line options -func projectFromOptions(options *ProjectOptions) (*Project, error) { +func ProjectFromOptions(options *ProjectOptions) (*Project, error) { configPath, err := getConfigPathFromOptions(options) if err != nil { return nil, err From 52440a473201d5e8391f301b1161f2f80265a14b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 28 Apr 2020 10:47:03 +0200 Subject: [PATCH 025/205] Setup Github Action for CI close #1 Signed-off-by: Nicolas De Loof --- ecs/Makefile | 5 ++++- ecs/golangci.yaml | 12 ++++++++++++ ecs/pkg/amazon/ecs.go | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 ecs/golangci.yaml diff --git a/ecs/Makefile b/ecs/Makefile index d8cc4a50..6ff9437d 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -7,4 +7,7 @@ test: ## Run tests dev: build ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" -.PHONY: build test dev \ No newline at end of file +lint: ## Verify Go files + golangci-lint run --config ./golangci.yaml ./... + +.PHONY: clean build test dev lint diff --git a/ecs/golangci.yaml b/ecs/golangci.yaml new file mode 100644 index 00000000..39629336 --- /dev/null +++ b/ecs/golangci.yaml @@ -0,0 +1,12 @@ +run: + deadline: 2m + +linters: + disable-all: true + enable: + - gofmt + - goimports + - golint + - gosimple + - ineffassign + - misspell \ No newline at end of file diff --git a/ecs/pkg/amazon/ecs.go b/ecs/pkg/amazon/ecs.go index 03ca92f4..a1ee2f92 100644 --- a/ecs/pkg/amazon/ecs.go +++ b/ecs/pkg/amazon/ecs.go @@ -2,6 +2,7 @@ package amazon import ( "errors" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ecs" "github.com/sirupsen/logrus" From 8c0fee5abf2237273ad9ba0b727ad545fcc69cbd Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 24 Apr 2020 14:19:14 +0200 Subject: [PATCH 026/205] Define amazon.API as a simplified and currated interface over AWS SDK This makes code simpler to read and easier to mock within tests Signed-off-by: Nicolas De Loof --- ecs/Makefile | 3 + ecs/pkg/amazon/api.go | 22 ++++ ecs/pkg/amazon/client.go | 28 +---- ecs/pkg/amazon/cloudformation.go | 42 ++++++- ecs/pkg/amazon/down.go | 14 +-- ecs/pkg/amazon/ecs.go | 50 --------- ecs/pkg/amazon/loadBalancer.go | 136 ---------------------- ecs/pkg/amazon/logs.go | 28 ----- ecs/pkg/amazon/network.go | 112 ------------------ ecs/pkg/amazon/roles.go | 49 -------- ecs/pkg/amazon/sdk.go | 187 +++++++++++++++++++++++++++++++ ecs/pkg/amazon/up.go | 47 +------- 12 files changed, 261 insertions(+), 457 deletions(-) create mode 100644 ecs/pkg/amazon/api.go delete mode 100644 ecs/pkg/amazon/ecs.go delete mode 100644 ecs/pkg/amazon/loadBalancer.go delete mode 100644 ecs/pkg/amazon/logs.go delete mode 100644 ecs/pkg/amazon/network.go delete mode 100644 ecs/pkg/amazon/roles.go create mode 100644 ecs/pkg/amazon/sdk.go diff --git a/ecs/Makefile b/ecs/Makefile index 6ff9437d..5d3c5883 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -1,3 +1,6 @@ +clean: + rm -rf dist/ + build: go build -v -o dist/docker-ecs cmd/main/main.go diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go new file mode 100644 index 00000000..954c1b45 --- /dev/null +++ b/ecs/pkg/amazon/api.go @@ -0,0 +1,22 @@ +package amazon + +import ( + "github.com/awslabs/goformation/v4/cloudformation" +) + +type API interface { + ClusterExists(name string) (bool, error) + CreateCluster(name string) (string, error) + DeleteCluster(name string) error + + GetDefaultVPC() (string, error) + GetSubNets(vpcId string) ([]string, error) + + ListRolesForPolicy(policy string) ([]string, error) + GetRoleArn(name string) (string, error) + + StackExists(name string) (bool, error) + CreateStack(name string, template *cloudformation.Template) error + DescribeStackEvents(stack string) error + DeleteStack(name string) error +} diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index ff202fc2..18ffc985 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -3,18 +3,6 @@ package amazon import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/aws/aws-sdk-go/service/ecs" - "github.com/aws/aws-sdk-go/service/ecs/ecsiface" - "github.com/aws/aws-sdk-go/service/elbv2" - "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" - "github.com/aws/aws-sdk-go/service/iam" - "github.com/aws/aws-sdk-go/service/iam/iamiface" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -35,26 +23,14 @@ func NewClient(profile string, cluster string, region string) (compose.API, erro return &client{ Cluster: cluster, Region: region, - sess: sess, - ECS: ecs.New(sess), - EC2: ec2.New(sess), - ELB: elbv2.New(sess), - CW: cloudwatchlogs.New(sess), - IAM: iam.New(sess), - CF: cloudformation.New(sess), + api: NewAPI(sess), }, nil } type client struct { Cluster string Region string - sess *session.Session - ECS ecsiface.ECSAPI - EC2 ec2iface.EC2API - ELB elbv2iface.ELBV2API - CW cloudwatchlogsiface.CloudWatchLogsAPI - IAM iamiface.IAMAPI - CF cloudformationiface.CloudFormationAPI + api API } var _ compose.API = &client{} diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index dd304ab4..7458e786 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -2,6 +2,8 @@ package amazon import ( "fmt" + "github.com/compose-spec/compose-go/types" + "github.com/sirupsen/logrus" "strings" ecsapi "github.com/aws/aws-sdk-go/service/ecs" @@ -14,12 +16,12 @@ import ( func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*cloudformation.Template, error) { template := cloudformation.NewTemplate() - vpc, err := c.GetDefaultVPC() + vpc, err := c.api.GetDefaultVPC() if err != nil { return nil, err } - subnets, err := c.GetSubNets(vpc) + subnets, err := c.api.GetSubNets(vpc) if err != nil { return nil, err } @@ -42,7 +44,7 @@ func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*clo GroupDescription: securityGroup, GroupName: securityGroup, SecurityGroupIngress: ingresses, - VpcId: *vpc, + VpcId: vpc, } for _, service := range project.Services { @@ -55,7 +57,7 @@ func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*clo if err != nil { return nil, err } - definition.TaskRoleArn = *role + definition.TaskRoleArn = role taskDefinition := fmt.Sprintf("%sTaskDefinition", service.Name) template.Resources[taskDefinition] = definition @@ -78,3 +80,35 @@ func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*clo } return template, nil } + +const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +var defaultTaskExecutionRole string + +// GetEcsTaskExecutionRole retrieve the role ARN to apply for task execution +func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (string, error) { + if arn, ok := spec.Extras["x-ecs-TaskExecutionRole"]; ok { + return arn.(string), nil + } + if defaultTaskExecutionRole != "" { + return defaultTaskExecutionRole, nil + } + + logrus.Debug("Retrieve Task Execution Role") + entities, err := c.api.ListRolesForPolicy(ECSTaskExecutionPolicy) + if err != nil { + return "", err + } + if len(entities) == 0 { + return "", fmt.Errorf("no Role is attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") + } + if len(entities) > 1 { + return "", fmt.Errorf("multiple Roles are attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") + } + + arn, err := c.api.GetRoleArn(entities[0]) + if err != nil { + return "", err + } + defaultTaskExecutionRole = arn + return arn, nil +} \ No newline at end of file diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index 4dfdff4a..49583864 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -2,30 +2,22 @@ package amazon import ( "fmt" - - "github.com/aws/aws-sdk-go/service/cloudformation" - cf "github.com/aws/aws-sdk-go/service/cloudformation" ) func (c *client) ComposeDown(projectName *string, keepLoadBalancer, deleteCluster bool) error { - _, err := c.CF.DeleteStack(&cloudformation.DeleteStackInput{ - StackName: projectName, - }) + err := c.api.DeleteStack(projectName) if err != nil { return err } fmt.Printf("Delete stack ") - if err = c.CF.WaitUntilStackDeleteComplete(&cf.DescribeStacksInput{StackName: projectName}); err != nil { - return err - } - fmt.Printf("... done.\n") + if !deleteCluster { return nil } fmt.Printf("Delete cluster %s", c.Cluster) - if err = c.DeleteCluster(); err != nil { + if err = c.api.DeleteCluster(c.Cluster); err != nil { return err } fmt.Printf("... done. \n") diff --git a/ecs/pkg/amazon/ecs.go b/ecs/pkg/amazon/ecs.go deleted file mode 100644 index a1ee2f92..00000000 --- a/ecs/pkg/amazon/ecs.go +++ /dev/null @@ -1,50 +0,0 @@ -package amazon - -import ( - "errors" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ecs" - "github.com/sirupsen/logrus" -) - -func (c client) RegisterTaskDefinition(task *ecs.RegisterTaskDefinitionInput) (*string, error) { - logrus.Debug("Register Task Definition") - def, err := c.ECS.RegisterTaskDefinition(task) - if err != nil { - return nil, err - } - return def.TaskDefinition.TaskDefinitionArn, err -} - -func (c client) CreateCluster() (*string, error) { - logrus.Debug("Create cluster ", c.Cluster) - response, err := c.ECS.CreateCluster(&ecs.CreateClusterInput{ClusterName: &c.Cluster}) - if err != nil { - return nil, err - } - return response.Cluster.Status, nil -} - -func (c client) DeleteCluster() error { - logrus.Debug("Delete cluster ", c.Cluster) - response, err := c.ECS.DeleteCluster(&ecs.DeleteClusterInput{Cluster: &c.Cluster}) - if err != nil { - return err - } - if *response.Cluster.Status == "INACTIVE" { - return nil - } - return errors.New("Failed to delete cluster, status: " + *response.Cluster.Status) -} - -func (c client) ClusterExists() (bool, error) { - logrus.Debug("Check if cluster was already created: ", c.Cluster) - clusters, err := c.ECS.DescribeClusters(&ecs.DescribeClustersInput{ - Clusters: []*string{aws.String(c.Cluster)}, - }) - if err != nil { - return false, err - } - return len(clusters.Clusters) > 0, nil -} diff --git a/ecs/pkg/amazon/loadBalancer.go b/ecs/pkg/amazon/loadBalancer.go deleted file mode 100644 index c8f96cd9..00000000 --- a/ecs/pkg/amazon/loadBalancer.go +++ /dev/null @@ -1,136 +0,0 @@ -package amazon - -import ( - "fmt" - "strings" - - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/sirupsen/logrus" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/elbv2" - "github.com/compose-spec/compose-go/types" -) - -func (c client) CreateLoadBalancer(project *compose.Project, subnets []*string) (*string, error) { - logrus.Debug("Create Load Balancer") - alb, err := c.ELB.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ - IpAddressType: nil, - Name: aws.String(fmt.Sprintf("%s-LoadBalancer", project.Name)), - Subnets: subnets, - Type: aws.String(elbv2.LoadBalancerTypeEnumNetwork), - Tags: []*elbv2.Tag{ - { - Key: aws.String("com.docker.compose.project"), - Value: aws.String(project.Name), - }, - }, - }) - if err != nil { - return nil, err - } - return alb.LoadBalancers[0].LoadBalancerArn, nil -} - -func (c client) DeleteLoadBalancer(project *compose.Project, keepLoadBalancer bool) error { - logrus.Debug("Delete Load Balancer") - // FIXME We can tag LoadBalancer but not search by tag ? - loadBalancer, err := c.ELB.DescribeLoadBalancers(&elbv2.DescribeLoadBalancersInput{ - Names: aws.StringSlice([]string{fmt.Sprintf("%s-LoadBalancer", project.Name)}), - }) - if err != nil { - return err - } - arn := loadBalancer.LoadBalancers[0].LoadBalancerArn - - err = c.DeleteListeners(arn) - if err != nil { - return err - } - - err = c.DeleteTargetGroups(arn) - if err != nil { - return err - } - - if !keepLoadBalancer { - _, err = c.ELB.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{LoadBalancerArn: arn}) - } - return err -} - -func (c client) CreateTargetGroup(name string, vpc *string, port types.ServicePortConfig) (*string, error) { - logrus.Debugf("Create Target Group %d/%s\n", port.Target, port.Protocol) - group, err := c.ELB.CreateTargetGroup(&elbv2.CreateTargetGroupInput{ - Name: aws.String(name), - Port: aws.Int64(int64(port.Target)), - Protocol: aws.String(strings.ToUpper(port.Protocol)), - TargetType: aws.String("ip"), - VpcId: vpc, - }) - if err != nil { - return nil, err - } - arn := group.TargetGroups[0].TargetGroupArn - return arn, nil -} - -func (c client) DeleteTargetGroups(loadBalancer *string) error { - groups, err := c.ELB.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{ - LoadBalancerArn: loadBalancer, - }) - if err != nil { - return err - } - for _, group := range groups.TargetGroups { - logrus.Debugf("Delete Target Group %s\n", *group.TargetGroupArn) - _, err := c.ELB.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{ - TargetGroupArn: group.TargetGroupArn, - }) - if err != nil { - return err - } - } - return nil -} - -func (c client) CreateListener(port types.ServicePortConfig, arn *string, target *string) error { - logrus.Debugf("Create Listener %d\n", port.Published) - _, err := c.ELB.CreateListener(&elbv2.CreateListenerInput{ - DefaultActions: []*elbv2.Action{ - { - ForwardConfig: &elbv2.ForwardActionConfig{ - TargetGroups: []*elbv2.TargetGroupTuple{ - { - TargetGroupArn: target, - }, - }, - }, - Type: aws.String(elbv2.ActionTypeEnumForward), - }, - }, - LoadBalancerArn: arn, - Port: aws.Int64(int64(port.Published)), - Protocol: aws.String(strings.ToUpper(port.Protocol)), - }) - return err -} - -func (c client) DeleteListeners(loadBalancer *string) error { - listeners, err := c.ELB.DescribeListeners(&elbv2.DescribeListenersInput{ - LoadBalancerArn: loadBalancer, - }) - if err != nil { - return err - } - for _, listener := range listeners.Listeners { - logrus.Debugf("Delete Listener %s\n", *listener.ListenerArn) - _, err := c.ELB.DeleteListener(&elbv2.DeleteListenerInput{ - ListenerArn: listener.ListenerArn, - }) - if err != nil { - return err - } - } - return nil -} diff --git a/ecs/pkg/amazon/logs.go b/ecs/pkg/amazon/logs.go deleted file mode 100644 index 0c06671c..00000000 --- a/ecs/pkg/amazon/logs.go +++ /dev/null @@ -1,28 +0,0 @@ -package amazon - -import ( - "fmt" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/sirupsen/logrus" -) - -// GetOrCreateLogGroup retrieve a pre-existing log group for project or create one -func (c client) GetOrCreateLogGroup(project *compose.Project) (*string, error) { - logrus.Debug("Create Log Group") - logGroup := fmt.Sprintf("/ecs/%s", project.Name) - _, err := c.CW.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{ - LogGroupName: aws.String(logGroup), - Tags: map[string]*string{ - ProjectTag: aws.String(project.Name), - }, - }) - if err != nil { - if _, ok := err.(*cloudwatchlogs.ResourceAlreadyExistsException); !ok { - return nil, err - } - } - return &logGroup, nil -} diff --git a/ecs/pkg/amazon/network.go b/ecs/pkg/amazon/network.go deleted file mode 100644 index 26b41275..00000000 --- a/ecs/pkg/amazon/network.go +++ /dev/null @@ -1,112 +0,0 @@ -package amazon - -import ( - "fmt" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/compose-spec/compose-go/types" - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/sirupsen/logrus" -) - -// GetDefaultVPC retrieve the default VPC for AWS account -func (c client) GetDefaultVPC() (*string, error) { - logrus.Debug("Retrieve default VPC") - vpcs, err := c.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("isDefault"), - Values: []*string{aws.String("true")}, - }, - }, - }) - if err != nil { - return nil, err - } - if len(vpcs.Vpcs) == 0 { - return nil, fmt.Errorf("account has not default VPC") - } - return vpcs.Vpcs[0].VpcId, nil -} - -// GetSubNets retrieve default subnets for a VPC -func (c client) GetSubNets(vpc *string) ([]string, error) { - logrus.Debug("Retrieve SubNets") - subnets, err := c.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ - DryRun: nil, - Filters: []*ec2.Filter{ - { - Name: aws.String("vpc-id"), - Values: []*string{vpc}, - }, - { - Name: aws.String("default-for-az"), - Values: []*string{aws.String("true")}, - }, - }, - }) - if err != nil { - return nil, err - } - - ids := []string{} - for _, subnet := range subnets.Subnets { - ids = append(ids, *subnet.SubnetId) - } - return ids, nil -} - -// CreateSecurityGroup create a security group for the project -func (c client) CreateSecurityGroup(project *compose.Project, vpc *string) (*string, error) { - logrus.Debug("Create Security Group") - name := fmt.Sprintf("%s Security Group", project.Name) - securityGroup, err := c.EC2.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ - Description: aws.String(name), - GroupName: aws.String(name), - VpcId: vpc, - }) - if err != nil { - return nil, err - } - - _, err = c.EC2.CreateTags(&ec2.CreateTagsInput{ - Resources: []*string{securityGroup.GroupId}, - Tags: []*ec2.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(name), - }, - { - Key: aws.String(ProjectTag), - Value: aws.String(project.Name), - }, - }, - }) - if err != nil { - return nil, err - } - - return securityGroup.GroupId, nil -} - -func (c *client) ExposePort(securityGroup *string, port types.ServicePortConfig) error { - logrus.Debugf("Authorize ingress port %d/%s\n", port.Published, port.Protocol) - _, err := c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ - GroupId: securityGroup, - IpPermissions: []*ec2.IpPermission{ - { - IpProtocol: aws.String(strings.ToUpper(port.Protocol)), - IpRanges: []*ec2.IpRange{ - { - CidrIp: aws.String("0.0.0.0/0"), - }, - }, - FromPort: aws.Int64(int64(port.Target)), - ToPort: aws.Int64(int64(port.Target)), - }, - }, - }) - return err -} diff --git a/ecs/pkg/amazon/roles.go b/ecs/pkg/amazon/roles.go deleted file mode 100644 index 7e4594af..00000000 --- a/ecs/pkg/amazon/roles.go +++ /dev/null @@ -1,49 +0,0 @@ -package amazon - -import ( - "fmt" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/iam" - "github.com/compose-spec/compose-go/types" - "github.com/sirupsen/logrus" -) - -const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" - -var defaultTaskExecutionRole *string - -// GetEcsTaskExecutionRole retrieve the role ARN to apply for task execution -func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (*string, error) { - if arn, ok := spec.Extras["x-ecs-TaskExecutionRole"]; ok { - s := arn.(string) - return &s, nil - } - if defaultTaskExecutionRole != nil { - return defaultTaskExecutionRole, nil - } - - logrus.Debug("Retrieve Task Execution Role") - entities, err := c.IAM.ListEntitiesForPolicy(&iam.ListEntitiesForPolicyInput{ - EntityFilter: aws.String("Role"), - PolicyArn: aws.String(ECSTaskExecutionPolicy), - }) - if err != nil { - return nil, err - } - if len(entities.PolicyRoles) == 0 { - return nil, fmt.Errorf("no Role is attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") - } - if len(entities.PolicyRoles) > 1 { - return nil, fmt.Errorf("multiple Roles are attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") - } - - role, err := c.IAM.GetRole(&iam.GetRoleInput{ - RoleName: entities.PolicyRoles[0].RoleName, - }) - if err != nil { - return nil, err - } - defaultTaskExecutionRole = role.Role.Arn - return role.Role.Arn, nil -} diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go new file mode 100644 index 00000000..228507c1 --- /dev/null +++ b/ecs/pkg/amazon/sdk.go @@ -0,0 +1,187 @@ +package amazon + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/aws/aws-sdk-go/service/ecs/ecsiface" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/iam/iamiface" + cf "github.com/awslabs/goformation/v4/cloudformation" + "github.com/sirupsen/logrus" +) + +type sdk struct { + sess *session.Session + ECS ecsiface.ECSAPI + EC2 ec2iface.EC2API + ELB elbv2iface.ELBV2API + CW cloudwatchlogsiface.CloudWatchLogsAPI + IAM iamiface.IAMAPI + CF cloudformationiface.CloudFormationAPI +} + +func NewAPI(sess *session.Session) API { + return sdk{ + ECS: ecs.New(sess), + EC2: ec2.New(sess), + ELB: elbv2.New(sess), + CW: cloudwatchlogs.New(sess), + IAM: iam.New(sess), + CF: cloudformation.New(sess), + } +} + +func (s sdk) ClusterExists(name string) (bool, error) { + logrus.Debug("Check if cluster was already created: ", name) + clusters, err := s.ECS.DescribeClusters(&ecs.DescribeClustersInput{ + Clusters: []*string{aws.String(name)}, + }) + if err != nil { + return false, err + } + return len(clusters.Clusters) > 0, nil +} + +func (s sdk) CreateCluster(name string) (string, error) { + logrus.Debug("Create cluster ", name) + response, err := s.ECS.CreateCluster(&ecs.CreateClusterInput{ClusterName: aws.String(name)}) + if err != nil { + return "", err + } + return *response.Cluster.Status, nil +} + +func (s sdk) DeleteCluster(name string) error { + logrus.Debug("Delete cluster ", name) + response, err := s.ECS.DeleteCluster(&ecs.DeleteClusterInput{Cluster: aws.String(name)}) + if err != nil { + return err + } + if *response.Cluster.Status == "INACTIVE" { + return nil + } + return fmt.Errorf("Failed to delete cluster, status: %s" + *response.Cluster.Status) +} + +func (s sdk) GetDefaultVPC() (string, error) { + logrus.Debug("Retrieve default VPC") + vpcs, err := s.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("isDefault"), + Values: []*string{aws.String("true")}, + }, + }, + }) + if err != nil { + return "", err + } + if len(vpcs.Vpcs) == 0 { + return "", fmt.Errorf("account has not default VPC") + } + return *vpcs.Vpcs[0].VpcId, nil +} + +func (s sdk) GetSubNets(vpc string) ([]string, error) { + logrus.Debug("Retrieve SubNets") + subnets, err := s.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ + DryRun: nil, + Filters: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{ aws.String(vpc)}, + }, + { + Name: aws.String("default-for-az"), + Values: []*string{aws.String("true")}, + }, + }, + }) + if err != nil { + return nil, err + } + + ids := []string{} + for _, subnet := range subnets.Subnets { + ids = append(ids, *subnet.SubnetId) + } + return ids, nil +} + +func (s sdk) ListRolesForPolicy(policy string) ([]string, error) { + entities, err := s.IAM.ListEntitiesForPolicy(&iam.ListEntitiesForPolicyInput{ + EntityFilter: aws.String("Role"), + PolicyArn: aws.String(policy), + }) + if err != nil { + return nil, err + } + roles := []string{} + for _, e := range entities.PolicyRoles { + roles = append(roles, *e.RoleName) + } + return roles, nil +} + +func (s sdk) GetRoleArn(name string) (string, error) { + role, err := s.IAM.GetRole(&iam.GetRoleInput{ + RoleName: aws.String(name), + }) + if err != nil { + return "", err + } + return *role.Role.Arn, nil +} + +func (s sdk) StackExists(name string) (bool, error) { + stacks, err := s.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ + StackName: aws.String(name), + }) + if err != nil { + // FIXME doesn't work as expected + return false, nil + } + return len(stacks.Stacks) > 0, nil +} + +func (s sdk) CreateStack(name string, template *cf.Template) error { + logrus.Debug("Create CloudFormation stack") + json, err := template.JSON() + if err != nil { + return err + } + + _, err = s.CF.CreateStack(&cloudformation.CreateStackInput{ + OnFailure: aws.String("DELETE"), + StackName: aws.String(name), + TemplateBody: aws.String(string(json)), + TimeoutInMinutes: aws.Int64(10), + }) + return err +} + +func (s sdk) DescribeStackEvents(name string) error { + // Fixme implement Paginator on Events and return as a chan(events) + _, err := s.CF.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{ + StackName: aws.String(name), + }) + return err +} + +func (s sdk) DeleteStack(name string) error { + logrus.Debug("Delete CloudFormation stack") + _, err := s.CF.DeleteStack(&cloudformation.DeleteStackInput{ + StackName: aws.String(name), + }) + return err +} diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 2dbfd306..4488cf67 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -2,26 +2,19 @@ package amazon import ( "fmt" - "os" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" ) func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) error { - ok, err := c.ClusterExists() + ok, err := c.api.ClusterExists(c.Cluster) if err != nil { return err } if !ok { - c.CreateCluster() + c.api.CreateCluster(c.Cluster) } - _, err = c.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: aws.String(project.Name), - }) - if err == nil { - // FIXME no ErrNotFound err type here + update, err := c.api.StackExists(project.Name) + if update { return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") } @@ -30,40 +23,12 @@ func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) er return err } - json, err := template.JSON() + err = c.api.CreateStack(project.Name, template) if err != nil { return err } - _, err = c.CF.ValidateTemplate(&cloudformation.ValidateTemplateInput{ - TemplateBody: aws.String(string(json)), - }) - if err != nil { - return err - } - - _, err = c.CF.CreateStack(&cloudformation.CreateStackInput{ - OnFailure: aws.String("DELETE"), - StackName: aws.String(project.Name), - TemplateBody: aws.String(string(json)), - TimeoutInMinutes: aws.Int64(10), - }) - if err != nil { - return err - } - - events, err := c.CF.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{ - StackName: aws.String(project.Name), - }) - if err != nil { - return err - } - for _, event := range events.StackEvents { - fmt.Printf("%s %s\n", *event.LogicalResourceId, *event.ResourceStatus) - if *event.ResourceStatus == "CREATE_FAILED" { - fmt.Fprintln(os.Stderr, event.ResourceStatusReason) - } - } + err = c.api.DescribeStackEvents(project.Name) // TODO monitor progress return nil From 52c6177ff7c91ece52b870373fccfb7bd6c50ad7 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 27 Apr 2020 09:42:44 +0200 Subject: [PATCH 027/205] API mock and a test case relying on it Signed-off-by: Nicolas De Loof --- ecs/go.mod | 1 + ecs/go.sum | 6 ++ ecs/pkg/amazon/down_test.go | 45 +++++++++ ecs/pkg/amazon/mock/api.go | 195 ++++++++++++++++++++++++++++++++++++ 4 files changed, 247 insertions(+) create mode 100644 ecs/pkg/amazon/down_test.go create mode 100644 ecs/pkg/amazon/mock/api.go diff --git a/ecs/go.mod b/ecs/go.mod index 0bc80e99..0ddf6c85 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -27,6 +27,7 @@ require ( github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.1 // indirect + github.com/golang/mock v1.4.3 github.com/gorilla/mux v1.7.3 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/jinzhu/gorm v1.9.12 // indirect diff --git a/ecs/go.sum b/ecs/go.sum index ee07c608..5cd52e76 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -119,6 +119,8 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -366,6 +368,7 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -378,6 +381,7 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -421,5 +425,7 @@ gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go new file mode 100644 index 00000000..3245ecef --- /dev/null +++ b/ecs/pkg/amazon/down_test.go @@ -0,0 +1,45 @@ +package amazon + +import ( + "github.com/docker/ecs-plugin/pkg/amazon/mock" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/golang/mock/gomock" + "testing" +) + +func Test_down_dont_delete_cluster(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + m := mock.NewMockAPI(ctrl) + c := &client{ + Cluster: "test_cluster", + Region: "region", + api: m, + } + + recorder := m.EXPECT() + recorder.DeleteStack("test_project").Return(nil).Times(1) + + c.ComposeDown(&compose.Project{ + Name: "test_project", + }, false, false) +} + +func Test_down_delete_cluster(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + m := mock.NewMockAPI(ctrl) + c := &client{ + Cluster: "test_cluster", + Region: "region", + api: m, + } + + recorder := m.EXPECT() + recorder.DeleteStack("test_project").Return(nil).Times(1) + recorder.DeleteCluster("test_cluster").Return(nil).Times(1) + + c.ComposeDown(&compose.Project{ + Name: "test_project", + }, false, true) +} \ No newline at end of file diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go new file mode 100644 index 00000000..8adaca5f --- /dev/null +++ b/ecs/pkg/amazon/mock/api.go @@ -0,0 +1,195 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./pkg/amazon/api.go + +// Package mock is a generated GoMock package. +package mock + +import ( + cloudformation "github.com/awslabs/goformation/v4/cloudformation" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockAPI is a mock of API interface +type MockAPI struct { + ctrl *gomock.Controller + recorder *MockAPIMockRecorder +} + +// MockAPIMockRecorder is the mock recorder for MockAPI +type MockAPIMockRecorder struct { + mock *MockAPI +} + +// NewMockAPI creates a new mock instance +func NewMockAPI(ctrl *gomock.Controller) *MockAPI { + mock := &MockAPI{ctrl: ctrl} + mock.recorder = &MockAPIMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockAPI) EXPECT() *MockAPIMockRecorder { + return m.recorder +} + +// ClusterExists mocks base method +func (m *MockAPI) ClusterExists(name string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClusterExists", name) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ClusterExists indicates an expected call of ClusterExists +func (mr *MockAPIMockRecorder) ClusterExists(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), name) +} + +// CreateCluster mocks base method +func (m *MockAPI) CreateCluster(name string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateCluster", name) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateCluster indicates an expected call of CreateCluster +func (mr *MockAPIMockRecorder) CreateCluster(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCluster", reflect.TypeOf((*MockAPI)(nil).CreateCluster), name) +} + +// DeleteCluster mocks base method +func (m *MockAPI) DeleteCluster(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCluster", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteCluster indicates an expected call of DeleteCluster +func (mr *MockAPIMockRecorder) DeleteCluster(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockAPI)(nil).DeleteCluster), name) +} + +// GetDefaultVPC mocks base method +func (m *MockAPI) GetDefaultVPC() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDefaultVPC") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDefaultVPC indicates an expected call of GetDefaultVPC +func (mr *MockAPIMockRecorder) GetDefaultVPC() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC)) +} + +// GetSubNets mocks base method +func (m *MockAPI) GetSubNets(vpcId string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSubNets", vpcId) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSubNets indicates an expected call of GetSubNets +func (mr *MockAPIMockRecorder) GetSubNets(vpcId interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), vpcId) +} + +// ListRolesForPolicy mocks base method +func (m *MockAPI) ListRolesForPolicy(policy string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRolesForPolicy", policy) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRolesForPolicy indicates an expected call of ListRolesForPolicy +func (mr *MockAPIMockRecorder) ListRolesForPolicy(policy interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRolesForPolicy", reflect.TypeOf((*MockAPI)(nil).ListRolesForPolicy), policy) +} + +// GetRoleArn mocks base method +func (m *MockAPI) GetRoleArn(name string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRoleArn", name) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRoleArn indicates an expected call of GetRoleArn +func (mr *MockAPIMockRecorder) GetRoleArn(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoleArn", reflect.TypeOf((*MockAPI)(nil).GetRoleArn), name) +} + +// StackExists mocks base method +func (m *MockAPI) StackExists(name string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StackExists", name) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StackExists indicates an expected call of StackExists +func (mr *MockAPIMockRecorder) StackExists(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StackExists", reflect.TypeOf((*MockAPI)(nil).StackExists), name) +} + +// CreateStack mocks base method +func (m *MockAPI) CreateStack(name string, template *cloudformation.Template) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateStack", name, template) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateStack indicates an expected call of CreateStack +func (mr *MockAPIMockRecorder) CreateStack(name, template interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), name, template) +} + +// DescribeStackEvents mocks base method +func (m *MockAPI) DescribeStackEvents(stack string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeStackEvents", stack) + ret0, _ := ret[0].(error) + return ret0 +} + +// DescribeStackEvents indicates an expected call of DescribeStackEvents +func (mr *MockAPIMockRecorder) DescribeStackEvents(stack interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), stack) +} + +// DeleteStack mocks base method +func (m *MockAPI) DeleteStack(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteStack", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteStack indicates an expected call of DeleteStack +func (mr *MockAPIMockRecorder) DeleteStack(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), name) +} From 3d8d982d4a547b7413c1833f551467b66cdd80f8 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 27 Apr 2020 09:46:13 +0200 Subject: [PATCH 028/205] format Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/client.go | 2 +- ecs/pkg/amazon/cloudformation.go | 3 ++- ecs/pkg/amazon/down_test.go | 2 +- ecs/pkg/amazon/sdk.go | 28 ++++++++++++++-------------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index 18ffc985..2f6fd433 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -30,7 +30,7 @@ func NewClient(profile string, cluster string, region string) (compose.API, erro type client struct { Cluster string Region string - api API + api API } var _ compose.API = &client{} diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 7458e786..91c8bca7 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -82,6 +82,7 @@ func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*clo } const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + var defaultTaskExecutionRole string // GetEcsTaskExecutionRole retrieve the role ARN to apply for task execution @@ -111,4 +112,4 @@ func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (string, error } defaultTaskExecutionRole = arn return arn, nil -} \ No newline at end of file +} diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go index 3245ecef..3738e78d 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/down_test.go @@ -42,4 +42,4 @@ func Test_down_delete_cluster(t *testing.T) { c.ComposeDown(&compose.Project{ Name: "test_project", }, false, true) -} \ No newline at end of file +} diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 228507c1..fdea7c6f 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -21,23 +21,23 @@ import ( ) type sdk struct { - sess *session.Session - ECS ecsiface.ECSAPI - EC2 ec2iface.EC2API - ELB elbv2iface.ELBV2API - CW cloudwatchlogsiface.CloudWatchLogsAPI - IAM iamiface.IAMAPI - CF cloudformationiface.CloudFormationAPI + sess *session.Session + ECS ecsiface.ECSAPI + EC2 ec2iface.EC2API + ELB elbv2iface.ELBV2API + CW cloudwatchlogsiface.CloudWatchLogsAPI + IAM iamiface.IAMAPI + CF cloudformationiface.CloudFormationAPI } func NewAPI(sess *session.Session) API { return sdk{ - ECS: ecs.New(sess), - EC2: ec2.New(sess), - ELB: elbv2.New(sess), - CW: cloudwatchlogs.New(sess), - IAM: iam.New(sess), - CF: cloudformation.New(sess), + ECS: ecs.New(sess), + EC2: ec2.New(sess), + ELB: elbv2.New(sess), + CW: cloudwatchlogs.New(sess), + IAM: iam.New(sess), + CF: cloudformation.New(sess), } } @@ -99,7 +99,7 @@ func (s sdk) GetSubNets(vpc string) ([]string, error) { Filters: []*ec2.Filter{ { Name: aws.String("vpc-id"), - Values: []*string{ aws.String(vpc)}, + Values: []*string{aws.String(vpc)}, }, { Name: aws.String("default-for-az"), From 4138dcfb5a1a0fbafc4bc824261265a026a96423 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 27 Apr 2020 10:04:07 +0200 Subject: [PATCH 029/205] Split API interface by required SDK func per command Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 4 ++-- ecs/pkg/amazon/api.go | 21 +++------------------ ecs/pkg/amazon/cloudformation.go | 12 ++++++++++-- ecs/pkg/amazon/down.go | 6 +++++- ecs/pkg/amazon/down_test.go | 7 ++++--- ecs/pkg/amazon/up.go | 14 ++++++++++++-- 6 files changed, 36 insertions(+), 28 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 7b3d1d6a..ad67ece0 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -98,7 +98,7 @@ func ConvertCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOpt if err != nil { return err } - template, err := client.Convert(project, opts.LoadBalancerArn()) + template, err := client.Convert(project) if err != nil { return err } @@ -124,7 +124,7 @@ func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) if err != nil { return err } - return client.ComposeUp(project, opts.LoadBalancerArn()) + return client.ComposeUp(project) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index 954c1b45..54301404 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -1,22 +1,7 @@ package amazon -import ( - "github.com/awslabs/goformation/v4/cloudformation" -) - type API interface { - ClusterExists(name string) (bool, error) - CreateCluster(name string) (string, error) - DeleteCluster(name string) error - - GetDefaultVPC() (string, error) - GetSubNets(vpcId string) ([]string, error) - - ListRolesForPolicy(policy string) ([]string, error) - GetRoleArn(name string) (string, error) - - StackExists(name string) (bool, error) - CreateStack(name string, template *cloudformation.Template) error - DescribeStackEvents(stack string) error - DeleteStack(name string) error + downAPI + upAPI + convertAPI } diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 91c8bca7..5f91d76e 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -2,9 +2,10 @@ package amazon import ( "fmt" + "strings" + "github.com/compose-spec/compose-go/types" "github.com/sirupsen/logrus" - "strings" ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" @@ -14,7 +15,7 @@ import ( "github.com/docker/ecs-plugin/pkg/convert" ) -func (c client) Convert(project *compose.Project, loadBalancerArn *string) (*cloudformation.Template, error) { +func (c client) Convert(project *compose.Project) (*cloudformation.Template, error) { template := cloudformation.NewTemplate() vpc, err := c.api.GetDefaultVPC() if err != nil { @@ -113,3 +114,10 @@ func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (string, error defaultTaskExecutionRole = arn return arn, nil } + +type convertAPI interface { + GetDefaultVPC() (string, error) + GetSubNets(vpcId string) ([]string, error) + ListRolesForPolicy(policy string) ([]string, error) + GetRoleArn(name string) (string, error) +} diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index 49583864..18557231 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -11,7 +11,6 @@ func (c *client) ComposeDown(projectName *string, keepLoadBalancer, deleteCluste } fmt.Printf("Delete stack ") - if !deleteCluster { return nil } @@ -23,3 +22,8 @@ func (c *client) ComposeDown(projectName *string, keepLoadBalancer, deleteCluste fmt.Printf("... done. \n") return nil } + +type downAPI interface { + DeleteStack(name string) error + DeleteCluster(name string) error +} diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go index 3738e78d..161bdfe1 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/down_test.go @@ -1,10 +1,11 @@ package amazon import ( + "testing" + "github.com/docker/ecs-plugin/pkg/amazon/mock" "github.com/docker/ecs-plugin/pkg/compose" "github.com/golang/mock/gomock" - "testing" ) func Test_down_dont_delete_cluster(t *testing.T) { @@ -22,7 +23,7 @@ func Test_down_dont_delete_cluster(t *testing.T) { c.ComposeDown(&compose.Project{ Name: "test_project", - }, false, false) + }, false) } func Test_down_delete_cluster(t *testing.T) { @@ -41,5 +42,5 @@ func Test_down_delete_cluster(t *testing.T) { c.ComposeDown(&compose.Project{ Name: "test_project", - }, false, true) + }, true) } diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 4488cf67..493c8cba 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -2,10 +2,12 @@ package amazon import ( "fmt" + + "github.com/awslabs/goformation/v4/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" ) -func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) error { +func (c *client) ComposeUp(project *compose.Project) error { ok, err := c.api.ClusterExists(c.Cluster) if err != nil { return err @@ -18,7 +20,7 @@ func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) er return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") } - template, err := c.Convert(project, loadBalancerArn) + template, err := c.Convert(project) if err != nil { return err } @@ -33,3 +35,11 @@ func (c *client) ComposeUp(project *compose.Project, loadBalancerArn *string) er // TODO monitor progress return nil } + +type upAPI interface { + ClusterExists(name string) (bool, error) + CreateCluster(name string) (string, error) + StackExists(name string) (bool, error) + CreateStack(name string, template *cloudformation.Template) error + DescribeStackEvents(stack string) error +} From 096c800c1b3a3389d3eeebaeba748cee0dca864f Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 27 Apr 2020 11:00:08 +0200 Subject: [PATCH 030/205] use go:generate to automatically run mockgen on build Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 1 - ecs/pkg/amazon/api.go | 2 + ecs/pkg/amazon/mock/api.go | 186 ++++++++++++++++++------------------- 3 files changed, 95 insertions(+), 94 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index ad67ece0..70468704 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -90,7 +90,6 @@ func (o upOptions) LoadBalancerArn() *string { } func ConvertCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { - opts := upOptions{} cmd := &cobra.Command{ Use: "convert", RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index 54301404..ff61174d 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -1,5 +1,7 @@ package amazon +//go:generate mockgen -destination=./mock/api.go -package=mock . API + type API interface { downAPI upAPI diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index 8adaca5f..ded7e8c8 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: ./pkg/amazon/api.go +// Source: github.com/docker/ecs-plugin/pkg/amazon (interfaces: API) // Package mock is a generated GoMock package. package mock @@ -34,47 +34,89 @@ func (m *MockAPI) EXPECT() *MockAPIMockRecorder { } // ClusterExists mocks base method -func (m *MockAPI) ClusterExists(name string) (bool, error) { +func (m *MockAPI) ClusterExists(arg0 string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClusterExists", name) + ret := m.ctrl.Call(m, "ClusterExists", arg0) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // ClusterExists indicates an expected call of ClusterExists -func (mr *MockAPIMockRecorder) ClusterExists(name interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) ClusterExists(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), name) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), arg0) } // CreateCluster mocks base method -func (m *MockAPI) CreateCluster(name string) (string, error) { +func (m *MockAPI) CreateCluster(arg0 string) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateCluster", name) + ret := m.ctrl.Call(m, "CreateCluster", arg0) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateCluster indicates an expected call of CreateCluster -func (mr *MockAPIMockRecorder) CreateCluster(name interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) CreateCluster(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCluster", reflect.TypeOf((*MockAPI)(nil).CreateCluster), name) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCluster", reflect.TypeOf((*MockAPI)(nil).CreateCluster), arg0) +} + +// CreateStack mocks base method +func (m *MockAPI) CreateStack(arg0 string, arg1 *cloudformation.Template) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateStack", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateStack indicates an expected call of CreateStack +func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1) } // DeleteCluster mocks base method -func (m *MockAPI) DeleteCluster(name string) error { +func (m *MockAPI) DeleteCluster(arg0 string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteCluster", name) + ret := m.ctrl.Call(m, "DeleteCluster", arg0) ret0, _ := ret[0].(error) return ret0 } // DeleteCluster indicates an expected call of DeleteCluster -func (mr *MockAPIMockRecorder) DeleteCluster(name interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) DeleteCluster(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockAPI)(nil).DeleteCluster), name) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockAPI)(nil).DeleteCluster), arg0) +} + +// DeleteStack mocks base method +func (m *MockAPI) DeleteStack(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteStack", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteStack indicates an expected call of DeleteStack +func (mr *MockAPIMockRecorder) DeleteStack(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), arg0) +} + +// DescribeStackEvents mocks base method +func (m *MockAPI) DescribeStackEvents(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeStackEvents", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// DescribeStackEvents indicates an expected call of DescribeStackEvents +func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0) } // GetDefaultVPC mocks base method @@ -92,104 +134,62 @@ func (mr *MockAPIMockRecorder) GetDefaultVPC() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC)) } -// GetSubNets mocks base method -func (m *MockAPI) GetSubNets(vpcId string) ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSubNets", vpcId) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetSubNets indicates an expected call of GetSubNets -func (mr *MockAPIMockRecorder) GetSubNets(vpcId interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), vpcId) -} - -// ListRolesForPolicy mocks base method -func (m *MockAPI) ListRolesForPolicy(policy string) ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListRolesForPolicy", policy) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListRolesForPolicy indicates an expected call of ListRolesForPolicy -func (mr *MockAPIMockRecorder) ListRolesForPolicy(policy interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRolesForPolicy", reflect.TypeOf((*MockAPI)(nil).ListRolesForPolicy), policy) -} - // GetRoleArn mocks base method -func (m *MockAPI) GetRoleArn(name string) (string, error) { +func (m *MockAPI) GetRoleArn(arg0 string) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRoleArn", name) + ret := m.ctrl.Call(m, "GetRoleArn", arg0) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetRoleArn indicates an expected call of GetRoleArn -func (mr *MockAPIMockRecorder) GetRoleArn(name interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) GetRoleArn(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoleArn", reflect.TypeOf((*MockAPI)(nil).GetRoleArn), name) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoleArn", reflect.TypeOf((*MockAPI)(nil).GetRoleArn), arg0) +} + +// GetSubNets mocks base method +func (m *MockAPI) GetSubNets(arg0 string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSubNets", arg0) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSubNets indicates an expected call of GetSubNets +func (mr *MockAPIMockRecorder) GetSubNets(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), arg0) +} + +// ListRolesForPolicy mocks base method +func (m *MockAPI) ListRolesForPolicy(arg0 string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRolesForPolicy", arg0) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRolesForPolicy indicates an expected call of ListRolesForPolicy +func (mr *MockAPIMockRecorder) ListRolesForPolicy(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRolesForPolicy", reflect.TypeOf((*MockAPI)(nil).ListRolesForPolicy), arg0) } // StackExists mocks base method -func (m *MockAPI) StackExists(name string) (bool, error) { +func (m *MockAPI) StackExists(arg0 string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StackExists", name) + ret := m.ctrl.Call(m, "StackExists", arg0) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // StackExists indicates an expected call of StackExists -func (mr *MockAPIMockRecorder) StackExists(name interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) StackExists(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StackExists", reflect.TypeOf((*MockAPI)(nil).StackExists), name) -} - -// CreateStack mocks base method -func (m *MockAPI) CreateStack(name string, template *cloudformation.Template) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateStack", name, template) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateStack indicates an expected call of CreateStack -func (mr *MockAPIMockRecorder) CreateStack(name, template interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), name, template) -} - -// DescribeStackEvents mocks base method -func (m *MockAPI) DescribeStackEvents(stack string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeStackEvents", stack) - ret0, _ := ret[0].(error) - return ret0 -} - -// DescribeStackEvents indicates an expected call of DescribeStackEvents -func (mr *MockAPIMockRecorder) DescribeStackEvents(stack interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), stack) -} - -// DeleteStack mocks base method -func (m *MockAPI) DeleteStack(name string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteStack", name) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteStack indicates an expected call of DeleteStack -func (mr *MockAPIMockRecorder) DeleteStack(name interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), name) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StackExists", reflect.TypeOf((*MockAPI)(nil).StackExists), arg0) } From 541bda3af8e4e8eb74e5ecb9b7d0032583492b09 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 28 Apr 2020 10:22:29 +0200 Subject: [PATCH 031/205] Remove ALB related options to be defined on phase 2 Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 8 +++----- ecs/pkg/amazon/down.go | 2 +- ecs/pkg/compose/api.go | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 70468704..85802552 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -131,8 +131,7 @@ func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) } type downOptions struct { - KeepLoadBalancer bool - DeleteCluster bool + DeleteCluster bool } func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { @@ -149,11 +148,11 @@ func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOption if err != nil { return err } - return client.ComposeDown(&project.Name, opts.KeepLoadBalancer, opts.DeleteCluster) + return client.ComposeDown(project.Name, opts.DeleteCluster) } // project names passed as parameters for _, name := range args { - err := client.ComposeDown(&name, opts.KeepLoadBalancer, opts.DeleteCluster) + err := client.ComposeDown(name, opts.DeleteCluster) if err != nil { return err } @@ -161,7 +160,6 @@ func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOption return nil }, } - cmd.Flags().BoolVar(&opts.KeepLoadBalancer, "keep-load-balancer", false, "Keep Load Balancer for further use") cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") return cmd } diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index 18557231..5c52e928 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -4,7 +4,7 @@ import ( "fmt" ) -func (c *client) ComposeDown(projectName *string, keepLoadBalancer, deleteCluster bool) error { +func (c *client) ComposeDown(projectName string, deleteCluster bool) error { err := c.api.DeleteStack(projectName) if err != nil { return err diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index b3075082..d2e2b1a2 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -3,7 +3,7 @@ package compose import "github.com/awslabs/goformation/v4/cloudformation" type API interface { - Convert(project *Project, loadBalancerArn *string) (*cloudformation.Template, error) - ComposeUp(project *Project, loadBalancerArn *string) error - ComposeDown(projectName *string, keepLoadBalancer, deleteCluster bool) error + Convert(project *Project) (*cloudformation.Template, error) + ComposeUp(project *Project) error + ComposeDown(projectName string, deleteCluster bool) error } From 30029fa701df1351e9d68a0e26516b4f07cc756a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 28 Apr 2020 10:37:35 +0200 Subject: [PATCH 032/205] ComposeDown only require stack name Signed-off-by: Nicolas De Loof g Sur la branche api Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/down_test.go | 9 ++------- ecs/pkg/amazon/sdk.go | 1 + ecs/pkg/compose/project_test.go | 8 ++++---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go index 161bdfe1..7e10c941 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/down_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/docker/ecs-plugin/pkg/amazon/mock" - "github.com/docker/ecs-plugin/pkg/compose" "github.com/golang/mock/gomock" ) @@ -21,9 +20,7 @@ func Test_down_dont_delete_cluster(t *testing.T) { recorder := m.EXPECT() recorder.DeleteStack("test_project").Return(nil).Times(1) - c.ComposeDown(&compose.Project{ - Name: "test_project", - }, false) + c.ComposeDown("test_project", false) } func Test_down_delete_cluster(t *testing.T) { @@ -40,7 +37,5 @@ func Test_down_delete_cluster(t *testing.T) { recorder.DeleteStack("test_project").Return(nil).Times(1) recorder.DeleteCluster("test_cluster").Return(nil).Times(1) - c.ComposeDown(&compose.Project{ - Name: "test_project", - }, true) + c.ComposeDown("test_project", true) } diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index fdea7c6f..75e3cc22 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -2,6 +2,7 @@ package amazon import ( "fmt" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" diff --git a/ecs/pkg/compose/project_test.go b/ecs/pkg/compose/project_test.go index 0b44fb33..2906c9d2 100644 --- a/ecs/pkg/compose/project_test.go +++ b/ecs/pkg/compose/project_test.go @@ -8,14 +8,14 @@ import ( ) func Test_project_name(t *testing.T) { - p, err := projectFromOptions(&ProjectOptions{ + p, err := ProjectFromOptions(&ProjectOptions{ name: "my_project", ConfigPaths: []string{"testdata/simple/compose.yaml"}, }) assert.NilError(t, err) assert.Equal(t, p.Name, "my_project") - p, err = projectFromOptions(&ProjectOptions{ + p, err = ProjectFromOptions(&ProjectOptions{ name: "", ConfigPaths: []string{"testdata/simple/compose.yaml"}, }) @@ -23,7 +23,7 @@ func Test_project_name(t *testing.T) { assert.Equal(t, p.Name, "simple") os.Setenv("COMPOSE_PROJECT_NAME", "my_project_from_env") - p, err = projectFromOptions(&ProjectOptions{ + p, err = ProjectFromOptions(&ProjectOptions{ name: "", ConfigPaths: []string{"testdata/simple/compose.yaml"}, }) @@ -32,7 +32,7 @@ func Test_project_name(t *testing.T) { } func Test_project_from_set_of_files(t *testing.T) { - p, err := projectFromOptions(&ProjectOptions{ + p, err := ProjectFromOptions(&ProjectOptions{ name: "my_project", ConfigPaths: []string{ "testdata/simple/compose.yaml", From 4642bfa1728888575cccab854804a3592391059b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 28 Apr 2020 11:05:00 +0200 Subject: [PATCH 033/205] Fix linter warnings Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 2 +- ecs/pkg/amazon/down_test.go | 4 ++-- ecs/pkg/amazon/sdk.go | 4 ++-- ecs/pkg/amazon/up.go | 6 ++++++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 5f91d76e..22b0c1d7 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -117,7 +117,7 @@ func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (string, error type convertAPI interface { GetDefaultVPC() (string, error) - GetSubNets(vpcId string) ([]string, error) + GetSubNets(vpcID string) ([]string, error) ListRolesForPolicy(policy string) ([]string, error) GetRoleArn(name string) (string, error) } diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go index 7e10c941..c0ec4d0e 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/down_test.go @@ -7,7 +7,7 @@ import ( "github.com/golang/mock/gomock" ) -func Test_down_dont_delete_cluster(t *testing.T) { +func TestDownDontDeleteCluster(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := mock.NewMockAPI(ctrl) @@ -23,7 +23,7 @@ func Test_down_dont_delete_cluster(t *testing.T) { c.ComposeDown("test_project", false) } -func Test_down_delete_cluster(t *testing.T) { +func TestDownDeleteCluster(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := mock.NewMockAPI(ctrl) diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 75e3cc22..86b65e34 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -93,14 +93,14 @@ func (s sdk) GetDefaultVPC() (string, error) { return *vpcs.Vpcs[0].VpcId, nil } -func (s sdk) GetSubNets(vpc string) ([]string, error) { +func (s sdk) GetSubNets(vpcID string) ([]string, error) { logrus.Debug("Retrieve SubNets") subnets, err := s.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ DryRun: nil, Filters: []*ec2.Filter{ { Name: aws.String("vpc-id"), - Values: []*string{aws.String(vpc)}, + Values: []*string{aws.String(vpcID)}, }, { Name: aws.String("default-for-az"), diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 493c8cba..d89385bc 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -16,6 +16,9 @@ func (c *client) ComposeUp(project *compose.Project) error { c.api.CreateCluster(c.Cluster) } update, err := c.api.StackExists(project.Name) + if err != nil { + return err + } if update { return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") } @@ -31,6 +34,9 @@ func (c *client) ComposeUp(project *compose.Project) error { } err = c.api.DescribeStackEvents(project.Name) + if err != nil { + return err + } // TODO monitor progress return nil From b6be4a0ac3626eaaafeaba546f3d40834ae522b5 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 28 Apr 2020 11:31:41 +0200 Subject: [PATCH 034/205] Use `WithContext` SDK APIs so we can implement cancelation Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 9 ++-- ecs/pkg/amazon/cloudformation.go | 23 +++++---- ecs/pkg/amazon/down.go | 11 ++-- ecs/pkg/amazon/down_test.go | 14 ++--- ecs/pkg/amazon/mock/api.go | 89 ++++++++++++++++---------------- ecs/pkg/amazon/sdk.go | 45 ++++++++-------- ecs/pkg/amazon/up.go | 25 ++++----- ecs/pkg/compose/api.go | 12 +++-- 8 files changed, 120 insertions(+), 108 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 85802552..99f556c7 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "github.com/docker/cli/cli-plugins/manager" @@ -97,7 +98,7 @@ func ConvertCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOpt if err != nil { return err } - template, err := client.Convert(project) + template, err := client.Convert(context.Background(), project) if err != nil { return err } @@ -123,7 +124,7 @@ func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) if err != nil { return err } - return client.ComposeUp(project) + return client.ComposeUp(context.Background(), project) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") @@ -148,11 +149,11 @@ func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOption if err != nil { return err } - return client.ComposeDown(project.Name, opts.DeleteCluster) + return client.ComposeDown(context.Background(), project.Name, opts.DeleteCluster) } // project names passed as parameters for _, name := range args { - err := client.ComposeDown(name, opts.DeleteCluster) + err := client.ComposeDown(context.Background(), name, opts.DeleteCluster) if err != nil { return err } diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 22b0c1d7..44a952fe 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -1,6 +1,7 @@ package amazon import ( + "context" "fmt" "strings" @@ -15,14 +16,14 @@ import ( "github.com/docker/ecs-plugin/pkg/convert" ) -func (c client) Convert(project *compose.Project) (*cloudformation.Template, error) { +func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudformation.Template, error) { template := cloudformation.NewTemplate() - vpc, err := c.api.GetDefaultVPC() + vpc, err := c.api.GetDefaultVPC(ctx) if err != nil { return nil, err } - subnets, err := c.api.GetSubNets(vpc) + subnets, err := c.api.GetSubNets(ctx, vpc) if err != nil { return nil, err } @@ -54,7 +55,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return nil, err } - role, err := c.GetEcsTaskExecutionRole(service) + role, err := c.GetEcsTaskExecutionRole(ctx, service) if err != nil { return nil, err } @@ -87,7 +88,7 @@ const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTa var defaultTaskExecutionRole string // GetEcsTaskExecutionRole retrieve the role ARN to apply for task execution -func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (string, error) { +func (c client) GetEcsTaskExecutionRole(ctx context.Context, spec types.ServiceConfig) (string, error) { if arn, ok := spec.Extras["x-ecs-TaskExecutionRole"]; ok { return arn.(string), nil } @@ -96,7 +97,7 @@ func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (string, error } logrus.Debug("Retrieve Task Execution Role") - entities, err := c.api.ListRolesForPolicy(ECSTaskExecutionPolicy) + entities, err := c.api.ListRolesForPolicy(ctx, ECSTaskExecutionPolicy) if err != nil { return "", err } @@ -107,7 +108,7 @@ func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (string, error return "", fmt.Errorf("multiple Roles are attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") } - arn, err := c.api.GetRoleArn(entities[0]) + arn, err := c.api.GetRoleArn(ctx, entities[0]) if err != nil { return "", err } @@ -116,8 +117,8 @@ func (c client) GetEcsTaskExecutionRole(spec types.ServiceConfig) (string, error } type convertAPI interface { - GetDefaultVPC() (string, error) - GetSubNets(vpcID string) ([]string, error) - ListRolesForPolicy(policy string) ([]string, error) - GetRoleArn(name string) (string, error) + GetDefaultVPC(ctx context.Context) (string, error) + GetSubNets(ctx context.Context, vpcID string) ([]string, error) + ListRolesForPolicy(ctx context.Context, policy string) ([]string, error) + GetRoleArn(ctx context.Context, name string) (string, error) } diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index 5c52e928..ab708d21 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -1,11 +1,12 @@ package amazon import ( + "context" "fmt" ) -func (c *client) ComposeDown(projectName string, deleteCluster bool) error { - err := c.api.DeleteStack(projectName) +func (c *client) ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error { + err := c.api.DeleteStack(ctx, projectName) if err != nil { return err } @@ -16,7 +17,7 @@ func (c *client) ComposeDown(projectName string, deleteCluster bool) error { } fmt.Printf("Delete cluster %s", c.Cluster) - if err = c.api.DeleteCluster(c.Cluster); err != nil { + if err = c.api.DeleteCluster(ctx, c.Cluster); err != nil { return err } fmt.Printf("... done. \n") @@ -24,6 +25,6 @@ func (c *client) ComposeDown(projectName string, deleteCluster bool) error { } type downAPI interface { - DeleteStack(name string) error - DeleteCluster(name string) error + DeleteStack(ctx context.Context, name string) error + DeleteCluster(ctx context.Context, name string) error } diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go index c0ec4d0e..3bfc1e40 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/down_test.go @@ -1,6 +1,7 @@ package amazon import ( + "context" "testing" "github.com/docker/ecs-plugin/pkg/amazon/mock" @@ -16,11 +17,11 @@ func TestDownDontDeleteCluster(t *testing.T) { Region: "region", api: m, } - + ctx := context.TODO() recorder := m.EXPECT() - recorder.DeleteStack("test_project").Return(nil).Times(1) + recorder.DeleteStack(ctx, "test_project").Return(nil).Times(1) - c.ComposeDown("test_project", false) + c.ComposeDown(ctx, "test_project", false) } func TestDownDeleteCluster(t *testing.T) { @@ -33,9 +34,10 @@ func TestDownDeleteCluster(t *testing.T) { api: m, } + ctx := context.TODO() recorder := m.EXPECT() - recorder.DeleteStack("test_project").Return(nil).Times(1) - recorder.DeleteCluster("test_cluster").Return(nil).Times(1) + recorder.DeleteStack(ctx, "test_project").Return(nil).Times(1) + recorder.DeleteCluster(ctx, "test_cluster").Return(nil).Times(1) - c.ComposeDown("test_project", true) + c.ComposeDown(ctx, "test_project", true) } diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index ded7e8c8..59f66177 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -5,6 +5,7 @@ package mock import ( + context "context" cloudformation "github.com/awslabs/goformation/v4/cloudformation" gomock "github.com/golang/mock/gomock" reflect "reflect" @@ -34,162 +35,162 @@ func (m *MockAPI) EXPECT() *MockAPIMockRecorder { } // ClusterExists mocks base method -func (m *MockAPI) ClusterExists(arg0 string) (bool, error) { +func (m *MockAPI) ClusterExists(arg0 context.Context, arg1 string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClusterExists", arg0) + ret := m.ctrl.Call(m, "ClusterExists", arg0, arg1) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // ClusterExists indicates an expected call of ClusterExists -func (mr *MockAPIMockRecorder) ClusterExists(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) ClusterExists(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), arg0, arg1) } // CreateCluster mocks base method -func (m *MockAPI) CreateCluster(arg0 string) (string, error) { +func (m *MockAPI) CreateCluster(arg0 context.Context, arg1 string) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateCluster", arg0) + ret := m.ctrl.Call(m, "CreateCluster", arg0, arg1) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateCluster indicates an expected call of CreateCluster -func (mr *MockAPIMockRecorder) CreateCluster(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) CreateCluster(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCluster", reflect.TypeOf((*MockAPI)(nil).CreateCluster), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCluster", reflect.TypeOf((*MockAPI)(nil).CreateCluster), arg0, arg1) } // CreateStack mocks base method -func (m *MockAPI) CreateStack(arg0 string, arg1 *cloudformation.Template) error { +func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation.Template) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateStack", arg0, arg1) + ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // CreateStack indicates an expected call of CreateStack -func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2) } // DeleteCluster mocks base method -func (m *MockAPI) DeleteCluster(arg0 string) error { +func (m *MockAPI) DeleteCluster(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteCluster", arg0) + ret := m.ctrl.Call(m, "DeleteCluster", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // DeleteCluster indicates an expected call of DeleteCluster -func (mr *MockAPIMockRecorder) DeleteCluster(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) DeleteCluster(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockAPI)(nil).DeleteCluster), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockAPI)(nil).DeleteCluster), arg0, arg1) } // DeleteStack mocks base method -func (m *MockAPI) DeleteStack(arg0 string) error { +func (m *MockAPI) DeleteStack(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteStack", arg0) + ret := m.ctrl.Call(m, "DeleteStack", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // DeleteStack indicates an expected call of DeleteStack -func (mr *MockAPIMockRecorder) DeleteStack(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) DeleteStack(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), arg0, arg1) } // DescribeStackEvents mocks base method -func (m *MockAPI) DescribeStackEvents(arg0 string) error { +func (m *MockAPI) DescribeStackEvents(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeStackEvents", arg0) + ret := m.ctrl.Call(m, "DescribeStackEvents", arg0, arg1) ret0, _ := ret[0].(error) return ret0 } // DescribeStackEvents indicates an expected call of DescribeStackEvents -func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0, arg1) } // GetDefaultVPC mocks base method -func (m *MockAPI) GetDefaultVPC() (string, error) { +func (m *MockAPI) GetDefaultVPC(arg0 context.Context) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetDefaultVPC") + ret := m.ctrl.Call(m, "GetDefaultVPC", arg0) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetDefaultVPC indicates an expected call of GetDefaultVPC -func (mr *MockAPIMockRecorder) GetDefaultVPC() *gomock.Call { +func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC), arg0) } // GetRoleArn mocks base method -func (m *MockAPI) GetRoleArn(arg0 string) (string, error) { +func (m *MockAPI) GetRoleArn(arg0 context.Context, arg1 string) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRoleArn", arg0) + ret := m.ctrl.Call(m, "GetRoleArn", arg0, arg1) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetRoleArn indicates an expected call of GetRoleArn -func (mr *MockAPIMockRecorder) GetRoleArn(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) GetRoleArn(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoleArn", reflect.TypeOf((*MockAPI)(nil).GetRoleArn), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoleArn", reflect.TypeOf((*MockAPI)(nil).GetRoleArn), arg0, arg1) } // GetSubNets mocks base method -func (m *MockAPI) GetSubNets(arg0 string) ([]string, error) { +func (m *MockAPI) GetSubNets(arg0 context.Context, arg1 string) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSubNets", arg0) + ret := m.ctrl.Call(m, "GetSubNets", arg0, arg1) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetSubNets indicates an expected call of GetSubNets -func (mr *MockAPIMockRecorder) GetSubNets(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) GetSubNets(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), arg0, arg1) } // ListRolesForPolicy mocks base method -func (m *MockAPI) ListRolesForPolicy(arg0 string) ([]string, error) { +func (m *MockAPI) ListRolesForPolicy(arg0 context.Context, arg1 string) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListRolesForPolicy", arg0) + ret := m.ctrl.Call(m, "ListRolesForPolicy", arg0, arg1) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } // ListRolesForPolicy indicates an expected call of ListRolesForPolicy -func (mr *MockAPIMockRecorder) ListRolesForPolicy(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) ListRolesForPolicy(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRolesForPolicy", reflect.TypeOf((*MockAPI)(nil).ListRolesForPolicy), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRolesForPolicy", reflect.TypeOf((*MockAPI)(nil).ListRolesForPolicy), arg0, arg1) } // StackExists mocks base method -func (m *MockAPI) StackExists(arg0 string) (bool, error) { +func (m *MockAPI) StackExists(arg0 context.Context, arg1 string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StackExists", arg0) + ret := m.ctrl.Call(m, "StackExists", arg0, arg1) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } // StackExists indicates an expected call of StackExists -func (mr *MockAPIMockRecorder) StackExists(arg0 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) StackExists(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StackExists", reflect.TypeOf((*MockAPI)(nil).StackExists), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StackExists", reflect.TypeOf((*MockAPI)(nil).StackExists), arg0, arg1) } diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 86b65e34..e3460c0d 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -1,6 +1,7 @@ package amazon import ( + "context" "fmt" "github.com/aws/aws-sdk-go/aws" @@ -42,9 +43,9 @@ func NewAPI(sess *session.Session) API { } } -func (s sdk) ClusterExists(name string) (bool, error) { +func (s sdk) ClusterExists(ctx context.Context, name string) (bool, error) { logrus.Debug("Check if cluster was already created: ", name) - clusters, err := s.ECS.DescribeClusters(&ecs.DescribeClustersInput{ + clusters, err := s.ECS.DescribeClustersWithContext(aws.Context(ctx), &ecs.DescribeClustersInput{ Clusters: []*string{aws.String(name)}, }) if err != nil { @@ -53,18 +54,18 @@ func (s sdk) ClusterExists(name string) (bool, error) { return len(clusters.Clusters) > 0, nil } -func (s sdk) CreateCluster(name string) (string, error) { +func (s sdk) CreateCluster(ctx context.Context, name string) (string, error) { logrus.Debug("Create cluster ", name) - response, err := s.ECS.CreateCluster(&ecs.CreateClusterInput{ClusterName: aws.String(name)}) + response, err := s.ECS.CreateClusterWithContext(aws.Context(ctx), &ecs.CreateClusterInput{ClusterName: aws.String(name)}) if err != nil { return "", err } return *response.Cluster.Status, nil } -func (s sdk) DeleteCluster(name string) error { +func (s sdk) DeleteCluster(ctx context.Context, name string) error { logrus.Debug("Delete cluster ", name) - response, err := s.ECS.DeleteCluster(&ecs.DeleteClusterInput{Cluster: aws.String(name)}) + response, err := s.ECS.DeleteClusterWithContext(aws.Context(ctx), &ecs.DeleteClusterInput{Cluster: aws.String(name)}) if err != nil { return err } @@ -74,9 +75,9 @@ func (s sdk) DeleteCluster(name string) error { return fmt.Errorf("Failed to delete cluster, status: %s" + *response.Cluster.Status) } -func (s sdk) GetDefaultVPC() (string, error) { +func (s sdk) GetDefaultVPC(ctx context.Context) (string, error) { logrus.Debug("Retrieve default VPC") - vpcs, err := s.EC2.DescribeVpcs(&ec2.DescribeVpcsInput{ + vpcs, err := s.EC2.DescribeVpcsWithContext(aws.Context(ctx), &ec2.DescribeVpcsInput{ Filters: []*ec2.Filter{ { Name: aws.String("isDefault"), @@ -93,9 +94,9 @@ func (s sdk) GetDefaultVPC() (string, error) { return *vpcs.Vpcs[0].VpcId, nil } -func (s sdk) GetSubNets(vpcID string) ([]string, error) { +func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) { logrus.Debug("Retrieve SubNets") - subnets, err := s.EC2.DescribeSubnets(&ec2.DescribeSubnetsInput{ + subnets, err := s.EC2.DescribeSubnetsWithContext(aws.Context(ctx), &ec2.DescribeSubnetsInput{ DryRun: nil, Filters: []*ec2.Filter{ { @@ -119,8 +120,8 @@ func (s sdk) GetSubNets(vpcID string) ([]string, error) { return ids, nil } -func (s sdk) ListRolesForPolicy(policy string) ([]string, error) { - entities, err := s.IAM.ListEntitiesForPolicy(&iam.ListEntitiesForPolicyInput{ +func (s sdk) ListRolesForPolicy(ctx context.Context, policy string) ([]string, error) { + entities, err := s.IAM.ListEntitiesForPolicyWithContext(aws.Context(ctx), &iam.ListEntitiesForPolicyInput{ EntityFilter: aws.String("Role"), PolicyArn: aws.String(policy), }) @@ -134,8 +135,8 @@ func (s sdk) ListRolesForPolicy(policy string) ([]string, error) { return roles, nil } -func (s sdk) GetRoleArn(name string) (string, error) { - role, err := s.IAM.GetRole(&iam.GetRoleInput{ +func (s sdk) GetRoleArn(ctx context.Context, name string) (string, error) { + role, err := s.IAM.GetRoleWithContext(aws.Context(ctx), &iam.GetRoleInput{ RoleName: aws.String(name), }) if err != nil { @@ -144,8 +145,8 @@ func (s sdk) GetRoleArn(name string) (string, error) { return *role.Role.Arn, nil } -func (s sdk) StackExists(name string) (bool, error) { - stacks, err := s.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ +func (s sdk) StackExists(ctx context.Context, name string) (bool, error) { + stacks, err := s.CF.DescribeStacksWithContext(aws.Context(ctx), &cloudformation.DescribeStacksInput{ StackName: aws.String(name), }) if err != nil { @@ -155,14 +156,14 @@ func (s sdk) StackExists(name string) (bool, error) { return len(stacks.Stacks) > 0, nil } -func (s sdk) CreateStack(name string, template *cf.Template) error { +func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template) error { logrus.Debug("Create CloudFormation stack") json, err := template.JSON() if err != nil { return err } - _, err = s.CF.CreateStack(&cloudformation.CreateStackInput{ + _, err = s.CF.CreateStackWithContext(aws.Context(ctx), &cloudformation.CreateStackInput{ OnFailure: aws.String("DELETE"), StackName: aws.String(name), TemplateBody: aws.String(string(json)), @@ -171,17 +172,17 @@ func (s sdk) CreateStack(name string, template *cf.Template) error { return err } -func (s sdk) DescribeStackEvents(name string) error { +func (s sdk) DescribeStackEvents(ctx context.Context, name string) error { // Fixme implement Paginator on Events and return as a chan(events) - _, err := s.CF.DescribeStackEvents(&cloudformation.DescribeStackEventsInput{ + _, err := s.CF.DescribeStackEventsWithContext(aws.Context(ctx), &cloudformation.DescribeStackEventsInput{ StackName: aws.String(name), }) return err } -func (s sdk) DeleteStack(name string) error { +func (s sdk) DeleteStack(ctx context.Context, name string) error { logrus.Debug("Delete CloudFormation stack") - _, err := s.CF.DeleteStack(&cloudformation.DeleteStackInput{ + _, err := s.CF.DeleteStackWithContext(aws.Context(ctx), &cloudformation.DeleteStackInput{ StackName: aws.String(name), }) return err diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index d89385bc..41b4f5cb 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -1,21 +1,22 @@ package amazon import ( + "context" "fmt" "github.com/awslabs/goformation/v4/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" ) -func (c *client) ComposeUp(project *compose.Project) error { - ok, err := c.api.ClusterExists(c.Cluster) +func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error { + ok, err := c.api.ClusterExists(ctx, c.Cluster) if err != nil { return err } if !ok { - c.api.CreateCluster(c.Cluster) + c.api.CreateCluster(ctx, c.Cluster) } - update, err := c.api.StackExists(project.Name) + update, err := c.api.StackExists(ctx, project.Name) if err != nil { return err } @@ -23,17 +24,17 @@ func (c *client) ComposeUp(project *compose.Project) error { return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") } - template, err := c.Convert(project) + template, err := c.Convert(ctx, project) if err != nil { return err } - err = c.api.CreateStack(project.Name, template) + err = c.api.CreateStack(ctx, project.Name, template) if err != nil { return err } - err = c.api.DescribeStackEvents(project.Name) + err = c.api.DescribeStackEvents(ctx, project.Name) if err != nil { return err } @@ -43,9 +44,9 @@ func (c *client) ComposeUp(project *compose.Project) error { } type upAPI interface { - ClusterExists(name string) (bool, error) - CreateCluster(name string) (string, error) - StackExists(name string) (bool, error) - CreateStack(name string, template *cloudformation.Template) error - DescribeStackEvents(stack string) error + ClusterExists(ctx context.Context, name string) (bool, error) + CreateCluster(ctx context.Context, name string) (string, error) + StackExists(ctx context.Context, name string) (bool, error) + CreateStack(ctx context.Context, name string, template *cloudformation.Template) error + DescribeStackEvents(ctx context.Context, stack string) error } diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index d2e2b1a2..f5e45bc6 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -1,9 +1,13 @@ package compose -import "github.com/awslabs/goformation/v4/cloudformation" +import ( + "context" + + "github.com/awslabs/goformation/v4/cloudformation" +) type API interface { - Convert(project *Project) (*cloudformation.Template, error) - ComposeUp(project *Project) error - ComposeDown(projectName string, deleteCluster bool) error + Convert(ctx context.Context, project *Project) (*cloudformation.Template, error) + ComposeUp(ctx context.Context, project *Project) error + ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error } From 39a59ae55fe78ad2e639ddd854ad8edec945aff5 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 23 Apr 2020 17:18:51 +0200 Subject: [PATCH 035/205] Deploy with user-defined vpc id Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 99f556c7..9e7fbb94 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -81,6 +81,7 @@ func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { type upOptions struct { loadBalancerArn string + vpcID string } func (o upOptions) LoadBalancerArn() *string { @@ -89,6 +90,12 @@ func (o upOptions) LoadBalancerArn() *string { } return &o.loadBalancerArn } +func (o upOptions) GetVPC() *string { + if o.vpcID == "" { + return nil + } + return &o.vpcID +} func ConvertCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ @@ -128,6 +135,7 @@ func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") + cmd.Flags().StringVar(&opts.vpcID, "vpc-id", "", "") return cmd } From 95c88acfb48f48b561aaecf227185d17a7127e07 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 27 Apr 2020 19:14:32 +0200 Subject: [PATCH 036/205] Set existing vpc as default external network in the compose file Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 9e7fbb94..99f556c7 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -81,7 +81,6 @@ func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { type upOptions struct { loadBalancerArn string - vpcID string } func (o upOptions) LoadBalancerArn() *string { @@ -90,12 +89,6 @@ func (o upOptions) LoadBalancerArn() *string { } return &o.loadBalancerArn } -func (o upOptions) GetVPC() *string { - if o.vpcID == "" { - return nil - } - return &o.vpcID -} func ConvertCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ @@ -135,7 +128,6 @@ func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") - cmd.Flags().StringVar(&opts.vpcID, "vpc-id", "", "") return cmd } From cec44fbb7b07db4ba9fda046e7193239673996e9 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 28 Apr 2020 14:33:40 +0200 Subject: [PATCH 037/205] move to sdk Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 29 ++++++++++++++++++++++++++++- ecs/pkg/amazon/sdk.go | 6 ++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 44a952fe..32afb346 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -1,7 +1,11 @@ package amazon import ( +<<<<<<< HEAD "context" +======= + "errors" +>>>>>>> a0701b8... move to sdk "fmt" "strings" @@ -18,7 +22,7 @@ import ( func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudformation.Template, error) { template := cloudformation.NewTemplate() - vpc, err := c.api.GetDefaultVPC(ctx) + vpc, err := c.GetVPC(ctx, project) if err != nil { return nil, err } @@ -83,6 +87,28 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo return template, nil } +func (c client) GetVPC(project *compose.Project) (string, error) { + //check compose file for the default external network + if net, ok := project.Networks["default"]; ok { + if net.External.External { + vpc := net.Name + ok, err := c.api.VpcExists(vpc) + if err != nil { + return "", err + } + if !ok { + return "", errors.New("Vpc does not exist: " + vpc) + } + return vpc, nil + } + } + defaultVPC, err := c.api.GetDefaultVPC() + if err != nil { + return "", err + } + return defaultVPC, nil +} + const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" var defaultTaskExecutionRole string @@ -118,6 +144,7 @@ func (c client) GetEcsTaskExecutionRole(ctx context.Context, spec types.ServiceC type convertAPI interface { GetDefaultVPC(ctx context.Context) (string, error) + VpcExists(ctx context.Context, vpcID string) (bool, error) GetSubNets(ctx context.Context, vpcID string) ([]string, error) ListRolesForPolicy(ctx context.Context, policy string) ([]string, error) GetRoleArn(ctx context.Context, name string) (string, error) diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index e3460c0d..b02e0df8 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -75,6 +75,12 @@ func (s sdk) DeleteCluster(ctx context.Context, name string) error { return fmt.Errorf("Failed to delete cluster, status: %s" + *response.Cluster.Status) } +func (s sdk) VpcExists(ctx context.Context, vpcID string) (bool, error) { + logrus.Debug("Check if VPC exists: ", vpcID) + _, err := s.EC2.DescribeVpcsWithContext(aws.Context(ctx), &ec2.DescribeVpcsInput{VpcIds: []*string{&vpcID}}) + return err == nil, err +} + func (s sdk) GetDefaultVPC(ctx context.Context) (string, error) { logrus.Debug("Retrieve default VPC") vpcs, err := s.EC2.DescribeVpcsWithContext(aws.Context(ctx), &ec2.DescribeVpcsInput{ From 4a6fec63d21b6f98cbfcb298fc7861a99e88ec65 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 28 Apr 2020 15:11:12 +0200 Subject: [PATCH 038/205] yet another rebase Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 32afb346..e5c4cd71 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -1,11 +1,8 @@ package amazon import ( -<<<<<<< HEAD "context" -======= "errors" ->>>>>>> a0701b8... move to sdk "fmt" "strings" @@ -87,12 +84,12 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo return template, nil } -func (c client) GetVPC(project *compose.Project) (string, error) { +func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, error) { //check compose file for the default external network if net, ok := project.Networks["default"]; ok { if net.External.External { vpc := net.Name - ok, err := c.api.VpcExists(vpc) + ok, err := c.api.VpcExists(ctx, vpc) if err != nil { return "", err } @@ -102,7 +99,7 @@ func (c client) GetVPC(project *compose.Project) (string, error) { return vpc, nil } } - defaultVPC, err := c.api.GetDefaultVPC() + defaultVPC, err := c.api.GetDefaultVPC(ctx) if err != nil { return "", err } From de365f41e91019b9d0c8b179721c5870ca52b12b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 29 Apr 2020 15:13:25 +0200 Subject: [PATCH 039/205] Fix test by regenerating mock Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/mock/api.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index 59f66177..fd9b01c0 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -194,3 +194,18 @@ func (mr *MockAPIMockRecorder) StackExists(arg0, arg1 interface{}) *gomock.Call mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StackExists", reflect.TypeOf((*MockAPI)(nil).StackExists), arg0, arg1) } + +// VpcExists mocks base method +func (m *MockAPI) VpcExists(arg0 context.Context, arg1 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VpcExists", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// VpcExists indicates an expected call of VpcExists +func (mr *MockAPIMockRecorder) VpcExists(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VpcExists", reflect.TypeOf((*MockAPI)(nil).VpcExists), arg0, arg1) +} From 2ad9504d158586c18c69adb0d3bfd1e4a7ca3e4d Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 29 Apr 2020 16:54:24 +0200 Subject: [PATCH 040/205] add secret interface Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/main/main.go | 95 +++++++++++++++++++++++++++++++++++++++ ecs/pkg/amazon/api.go | 1 + ecs/pkg/amazon/sdk.go | 24 ++++++++++ ecs/pkg/amazon/secrets.go | 28 ++++++++++++ ecs/pkg/compose/api.go | 5 +++ 5 files changed, 153 insertions(+) create mode 100644 ecs/pkg/amazon/secrets.go diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 99f556c7..9326f3ee 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "github.com/docker/cli/cli-plugins/manager" @@ -45,6 +46,7 @@ func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { cmd.AddCommand( VersionCommand(), ComposeCommand(&opts), + SecretCommand(&opts), ) cmd.Flags().StringVarP(&opts.profile, "profile", "p", "default", "AWS Profile") cmd.Flags().StringVarP(&opts.cluster, "cluster", "c", "default", "ECS cluster") @@ -164,3 +166,96 @@ func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOption cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") return cmd } + +func SecretCommand(clusteropts *clusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "secret", + } + opts := &compose.ProjectOptions{} + opts.AddFlags(cmd.Flags()) + + cmd.AddCommand( + CreateSecret(clusteropts), + InspectSecret(clusteropts), + ListSecrets(clusteropts), + DeleteSecret(clusteropts), + ) + return cmd +} + +type createSecretOptions struct { + Label string +} + +func CreateSecret(clusteropts *clusterOptions) *cobra.Command { + //opts := createSecretOptions{} + cmd := &cobra.Command{ + Use: "create [NAME]", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) + if err != nil { + return err + } + if len(args) == 0 { + return errors.New("Missing mandatory parameter: [NAME]") + } + name := args[0] + content := "blabla" + id, err := client.CreateSecret(context.Background(), name, content) + fmt.Println(id) + return err + }, + } + //cmd.Flags().BoolVar(&opts.Label, "label", false, "Secret label") + return cmd +} + +func InspectSecret(clusteropts *clusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "inspect [NAME]", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) + if err != nil { + return err + } + if len(args) == 0 { + return errors.New("Missing mandatory parameter: [NAME]") + } + name := args[0] + return client.InspectSecret(context.Background(), name) + }, + } + return cmd +} + +func ListSecrets(clusteropts *clusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) + if err != nil { + return err + } + return client.ListSecrets(context.Background()) + }, + } + return cmd +} + +func DeleteSecret(clusteropts *clusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "delete [NAME]", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) + if err != nil { + return err + } + if len(args) == 0 { + return errors.New("Missing mandatory parameter: [NAME]") + } + return client.DeleteSecret(context.Background(), args[0]) + }, + } + return cmd +} diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index ff61174d..b4914d68 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -6,4 +6,5 @@ type API interface { downAPI upAPI convertAPI + secretsAPI } diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index b02e0df8..2aa017e9 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -18,6 +18,8 @@ import ( "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/iam/iamiface" + "github.com/aws/aws-sdk-go/service/secretsmanager" + "github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface" cf "github.com/awslabs/goformation/v4/cloudformation" "github.com/sirupsen/logrus" ) @@ -30,6 +32,7 @@ type sdk struct { CW cloudwatchlogsiface.CloudWatchLogsAPI IAM iamiface.IAMAPI CF cloudformationiface.CloudFormationAPI + SM secretsmanageriface.SecretsManagerAPI } func NewAPI(sess *session.Session) API { @@ -40,6 +43,7 @@ func NewAPI(sess *session.Session) API { CW: cloudwatchlogs.New(sess), IAM: iam.New(sess), CF: cloudformation.New(sess), + SM: secretsmanager.New(sess), } } @@ -193,3 +197,23 @@ func (s sdk) DeleteStack(ctx context.Context, name string) error { }) return err } + +func (s sdk) CreateSecret(ctx context.Context, name string, content string) (string, error) { + logrus.Debug("Create secret " + name) + return "test", nil +} + +func (s sdk) InspectSecret(ctx context.Context, name string) error { + fmt.Printf("... done. \n") + return nil +} + +func (s sdk) ListSecrets(ctx context.Context) error { + fmt.Printf("... done. \n") + return nil +} + +func (s sdk) DeleteSecret(ctx context.Context, name string) error { + fmt.Printf("... done. \n") + return nil +} diff --git a/ecs/pkg/amazon/secrets.go b/ecs/pkg/amazon/secrets.go new file mode 100644 index 00000000..daabe79b --- /dev/null +++ b/ecs/pkg/amazon/secrets.go @@ -0,0 +1,28 @@ +package amazon + +import ( + "context" +) + +type secretsAPI interface { + CreateSecret(ctx context.Context, name string, content string) (string, error) + InspectSecret(ctx context.Context, name string) error + ListSecrets(ctx context.Context) error + DeleteSecret(ctx context.Context, name string) error +} + +func (c client) CreateSecret(ctx context.Context, name string, content string) (string, error) { + return c.api.CreateSecret(ctx, name, content) +} + +func (c client) InspectSecret(ctx context.Context, name string) error { + return c.api.InspectSecret(ctx, name) +} + +func (c client) ListSecrets(ctx context.Context) error { + return c.api.ListSecrets(ctx) +} + +func (c client) DeleteSecret(ctx context.Context, name string) error { + return c.api.DeleteSecret(ctx, name) +} diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index f5e45bc6..32095102 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -10,4 +10,9 @@ type API interface { Convert(ctx context.Context, project *Project) (*cloudformation.Template, error) ComposeUp(ctx context.Context, project *Project) error ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error + + CreateSecret(ctx context.Context, name string, content string) (string, error) + InspectSecret(ctx context.Context, name string) error + ListSecrets(ctx context.Context) error + DeleteSecret(ctx context.Context, name string) error } From 41aaf802e39fbb3a5495093a77eeddb6f7606ebb Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 29 Apr 2020 20:36:00 +0200 Subject: [PATCH 041/205] implement secret management Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/Makefile | 2 +- ecs/cmd/commands/compose.go | 117 ++++++++++++++++ ecs/cmd/commands/secret.go | 147 ++++++++++++++++++++ ecs/cmd/main.go | 58 ++++++++ ecs/cmd/main/main.go | 261 ------------------------------------ ecs/pkg/amazon/sdk.go | 70 ++++++++-- ecs/pkg/amazon/secrets.go | 18 +-- ecs/pkg/compose/api.go | 9 +- ecs/pkg/docker/secret.go | 20 +++ 9 files changed, 417 insertions(+), 285 deletions(-) create mode 100644 ecs/cmd/commands/compose.go create mode 100644 ecs/cmd/commands/secret.go create mode 100644 ecs/cmd/main.go delete mode 100644 ecs/cmd/main/main.go create mode 100644 ecs/pkg/docker/secret.go diff --git a/ecs/Makefile b/ecs/Makefile index 5d3c5883..e07af815 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -2,7 +2,7 @@ clean: rm -rf dist/ build: - go build -v -o dist/docker-ecs cmd/main/main.go + go build -v -o dist/docker-ecs cmd/main.go test: ## Run tests go test ./... -v diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go new file mode 100644 index 00000000..972477b3 --- /dev/null +++ b/ecs/cmd/commands/compose.go @@ -0,0 +1,117 @@ +package commands + +import ( + "context" + "fmt" + + "github.com/docker/ecs-plugin/pkg/amazon" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/spf13/cobra" +) + +type ClusterOptions struct { + Profile string + Region string + Cluster string +} + +func ComposeCommand(clusteropts *ClusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "compose", + } + opts := &compose.ProjectOptions{} + opts.AddFlags(cmd.Flags()) + + cmd.AddCommand( + ConvertCommand(clusteropts, opts), + UpCommand(clusteropts, opts), + DownCommand(clusteropts, opts), + ) + return cmd +} + +type upOptions struct { + loadBalancerArn string +} + +func (o upOptions) LoadBalancerArn() *string { + if o.loadBalancerArn == "" { + return nil + } + return &o.loadBalancerArn +} + +func ConvertCommand(clusteropts *ClusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "convert", + RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + template, err := client.Convert(context.Background(), project) + if err != nil { + return err + } + + j, err := template.JSON() + if err != nil { + fmt.Printf("Failed to generate JSON: %s\n", err) + } else { + fmt.Printf("%s\n", string(j)) + } + return nil + }), + } + return cmd +} + +func UpCommand(clusteropts *ClusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { + opts := upOptions{} + cmd := &cobra.Command{ + Use: "up", + RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + return client.ComposeUp(context.Background(), project) + }), + } + cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") + return cmd +} + +type downOptions struct { + DeleteCluster bool +} + +func DownCommand(clusteropts *ClusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { + opts := downOptions{} + cmd := &cobra.Command{ + Use: "down", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + if len(args) == 0 { + project, err := compose.ProjectFromOptions(projectOpts) + if err != nil { + return err + } + return client.ComposeDown(context.Background(), project.Name, opts.DeleteCluster) + } + // project names passed as parameters + for _, name := range args { + err := client.ComposeDown(context.Background(), name, opts.DeleteCluster) + if err != nil { + return err + } + } + return nil + }, + } + cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") + return cmd +} diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go new file mode 100644 index 00000000..0c228c54 --- /dev/null +++ b/ecs/cmd/commands/secret.go @@ -0,0 +1,147 @@ +package commands + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + + "github.com/docker/ecs-plugin/pkg/amazon" + "github.com/docker/ecs-plugin/pkg/docker" + "github.com/spf13/cobra" +) + +type createSecretOptions struct { + Label string +} + +type deleteSecretOptions struct { + recover bool +} + +func SecretCommand(clusteropts *ClusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "secret", + Short: "Manages secrets", + } + + cmd.AddCommand( + CreateSecret(clusteropts), + InspectSecret(clusteropts), + ListSecrets(clusteropts), + DeleteSecret(clusteropts), + ) + return cmd +} + +func CreateSecret(clusteropts *ClusterOptions) *cobra.Command { + //opts := createSecretOptions{} + cmd := &cobra.Command{ + Use: "create NAME SECRET", + Short: "Creates a secret.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + if len(args) == 0 { + return errors.New("Missing mandatory parameter: NAME") + } + name := args[0] + secret := args[1] + id, err := client.CreateSecret(context.Background(), name, secret) + fmt.Println(id) + return err + }, + } + return cmd +} + +func InspectSecret(clusteropts *ClusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "inspect ID", + Short: "Displays secret details", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + if len(args) == 0 { + return errors.New("Missing mandatory parameter: ID") + } + id := args[0] + secret, err := client.InspectSecret(context.Background(), id) + if err != nil { + return err + } + out, err := secret.ToJSON() + if err != nil { + return err + } + fmt.Println(out) + return nil + }, + } + return cmd +} + +func ListSecrets(clusteropts *ClusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List secrets stored for the existing account.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + secrets, err := client.ListSecrets(context.Background()) + if err != nil { + return err + } + + printList(os.Stdout, secrets) + return nil + }, + } + return cmd +} + +func DeleteSecret(clusteropts *ClusterOptions) *cobra.Command { + opts := deleteSecretOptions{} + cmd := &cobra.Command{ + Use: "delete NAME", + Aliases: []string{"rm", "remove"}, + Short: "Removes a secret.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + if len(args) == 0 { + return errors.New("Missing mandatory parameter: [NAME]") + } + return client.DeleteSecret(context.Background(), args[0], opts.recover) + }, + } + cmd.Flags().BoolVar(&opts.recover, "recover", false, "Enable recovery.") + return cmd +} + +func printList(out io.Writer, secrets []docker.Secret) { + printSection(out, len(secrets), func(w io.Writer) { + for _, secret := range secrets { + fmt.Fprintf(w, "%s\t%s\t%s\n", secret.ID, secret.Name, secret.Description) + } + }, "ID", "NAME", "DESCRIPTION") +} + +func printSection(out io.Writer, len int, printer func(io.Writer), headers ...string) { + w := tabwriter.NewWriter(out, 20, 1, 3, ' ', 0) + fmt.Fprintln(w, strings.Join(headers, "\t")) + printer(w) + w.Flush() +} diff --git a/ecs/cmd/main.go b/ecs/cmd/main.go new file mode 100644 index 00000000..72d072a0 --- /dev/null +++ b/ecs/cmd/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "fmt" + + "github.com/docker/cli/cli-plugins/manager" + "github.com/docker/cli/cli-plugins/plugin" + "github.com/docker/cli/cli/command" + commands "github.com/docker/ecs-plugin/cmd/commands" + "github.com/spf13/cobra" +) + +const version = "0.0.1" + +func main() { + plugin.Run(func(dockerCli command.Cli) *cobra.Command { + cmd := NewRootCmd("ecs", dockerCli) + return cmd + }, manager.Metadata{ + SchemaVersion: "0.1.0", + Vendor: "Docker Inc.", + Version: version, + Experimental: true, + }) +} + +// NewRootCmd returns the base root command. +func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { + var opts commands.ClusterOptions + + cmd := &cobra.Command{ + Short: "Docker ECS", + Long: `run multi-container applications on Amazon ECS.`, + Use: name, + Annotations: map[string]string{"experimentalCLI": "true"}, + } + cmd.AddCommand( + VersionCommand(), + commands.ComposeCommand(&opts), + commands.SecretCommand(&opts), + ) + cmd.Flags().StringVarP(&opts.Profile, "profile", "p", "default", "AWS Profile") + cmd.Flags().StringVarP(&opts.Cluster, "cluster", "c", "default", "ECS cluster") + cmd.Flags().StringVarP(&opts.Region, "region", "r", "", "AWS region") + + return cmd +} + +func VersionCommand() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Show version.", + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Printf("Docker ECS plugin %s\n", version) + return nil + }, + } +} diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go deleted file mode 100644 index 9326f3ee..00000000 --- a/ecs/cmd/main/main.go +++ /dev/null @@ -1,261 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - - "github.com/docker/cli/cli-plugins/manager" - "github.com/docker/cli/cli-plugins/plugin" - "github.com/docker/cli/cli/command" - "github.com/docker/ecs-plugin/pkg/amazon" - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/spf13/cobra" -) - -const version = "0.0.1" - -func main() { - plugin.Run(func(dockerCli command.Cli) *cobra.Command { - cmd := NewRootCmd("ecs", dockerCli) - return cmd - }, manager.Metadata{ - SchemaVersion: "0.1.0", - Vendor: "Docker Inc.", - Version: version, - Experimental: true, - }) -} - -type clusterOptions struct { - profile string - region string - cluster string -} - -// NewRootCmd returns the base root command. -func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { - var opts clusterOptions - - cmd := &cobra.Command{ - Short: "Docker ECS", - Long: `run multi-container applications on Amazon ECS.`, - Use: name, - Annotations: map[string]string{"experimentalCLI": "true"}, - } - cmd.AddCommand( - VersionCommand(), - ComposeCommand(&opts), - SecretCommand(&opts), - ) - cmd.Flags().StringVarP(&opts.profile, "profile", "p", "default", "AWS Profile") - cmd.Flags().StringVarP(&opts.cluster, "cluster", "c", "default", "ECS cluster") - cmd.Flags().StringVarP(&opts.region, "region", "r", "", "AWS region") - - return cmd -} - -func VersionCommand() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Show version.", - RunE: func(cmd *cobra.Command, args []string) error { - fmt.Printf("Docker ECS plugin %s\n", version) - return nil - }, - } -} - -func ComposeCommand(clusteropts *clusterOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "compose", - } - opts := &compose.ProjectOptions{} - opts.AddFlags(cmd.Flags()) - - cmd.AddCommand( - ConvertCommand(clusteropts, opts), - UpCommand(clusteropts, opts), - DownCommand(clusteropts, opts), - ) - return cmd -} - -type upOptions struct { - loadBalancerArn string -} - -func (o upOptions) LoadBalancerArn() *string { - if o.loadBalancerArn == "" { - return nil - } - return &o.loadBalancerArn -} - -func ConvertCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "convert", - RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { - client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) - if err != nil { - return err - } - template, err := client.Convert(context.Background(), project) - if err != nil { - return err - } - - j, err := template.JSON() - if err != nil { - fmt.Printf("Failed to generate JSON: %s\n", err) - } else { - fmt.Printf("%s\n", string(j)) - } - return nil - }), - } - return cmd -} - -func UpCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { - opts := upOptions{} - cmd := &cobra.Command{ - Use: "up", - RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { - client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) - if err != nil { - return err - } - return client.ComposeUp(context.Background(), project) - }), - } - cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") - return cmd -} - -type downOptions struct { - DeleteCluster bool -} - -func DownCommand(clusteropts *clusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { - opts := downOptions{} - cmd := &cobra.Command{ - Use: "down", - RunE: func(cmd *cobra.Command, args []string) error { - client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) - if err != nil { - return err - } - if len(args) == 0 { - project, err := compose.ProjectFromOptions(projectOpts) - if err != nil { - return err - } - return client.ComposeDown(context.Background(), project.Name, opts.DeleteCluster) - } - // project names passed as parameters - for _, name := range args { - err := client.ComposeDown(context.Background(), name, opts.DeleteCluster) - if err != nil { - return err - } - } - return nil - }, - } - cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") - return cmd -} - -func SecretCommand(clusteropts *clusterOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "secret", - } - opts := &compose.ProjectOptions{} - opts.AddFlags(cmd.Flags()) - - cmd.AddCommand( - CreateSecret(clusteropts), - InspectSecret(clusteropts), - ListSecrets(clusteropts), - DeleteSecret(clusteropts), - ) - return cmd -} - -type createSecretOptions struct { - Label string -} - -func CreateSecret(clusteropts *clusterOptions) *cobra.Command { - //opts := createSecretOptions{} - cmd := &cobra.Command{ - Use: "create [NAME]", - RunE: func(cmd *cobra.Command, args []string) error { - client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) - if err != nil { - return err - } - if len(args) == 0 { - return errors.New("Missing mandatory parameter: [NAME]") - } - name := args[0] - content := "blabla" - id, err := client.CreateSecret(context.Background(), name, content) - fmt.Println(id) - return err - }, - } - //cmd.Flags().BoolVar(&opts.Label, "label", false, "Secret label") - return cmd -} - -func InspectSecret(clusteropts *clusterOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "inspect [NAME]", - RunE: func(cmd *cobra.Command, args []string) error { - client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) - if err != nil { - return err - } - if len(args) == 0 { - return errors.New("Missing mandatory parameter: [NAME]") - } - name := args[0] - return client.InspectSecret(context.Background(), name) - }, - } - return cmd -} - -func ListSecrets(clusteropts *clusterOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "list", - Aliases: []string{"ls"}, - RunE: func(cmd *cobra.Command, args []string) error { - client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) - if err != nil { - return err - } - return client.ListSecrets(context.Background()) - }, - } - return cmd -} - -func DeleteSecret(clusteropts *clusterOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "delete [NAME]", - RunE: func(cmd *cobra.Command, args []string) error { - client, err := amazon.NewClient(clusteropts.profile, clusteropts.cluster, clusteropts.region) - if err != nil { - return err - } - if len(args) == 0 { - return errors.New("Missing mandatory parameter: [NAME]") - } - return client.DeleteSecret(context.Background(), args[0]) - }, - } - return cmd -} diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 2aa017e9..8f8c3403 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -22,6 +22,8 @@ import ( "github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface" cf "github.com/awslabs/goformation/v4/cloudformation" "github.com/sirupsen/logrus" + + "github.com/docker/ecs-plugin/pkg/docker" ) type sdk struct { @@ -198,22 +200,68 @@ func (s sdk) DeleteStack(ctx context.Context, name string) error { return err } -func (s sdk) CreateSecret(ctx context.Context, name string, content string) (string, error) { +func (s sdk) CreateSecret(ctx context.Context, name string, secret string) (string, error) { logrus.Debug("Create secret " + name) - return "test", nil + response, err := s.SM.CreateSecret(&secretsmanager.CreateSecretInput{Name: &name, SecretString: &secret}) + if err != nil { + return "", err + } + return *response.ARN, nil } -func (s sdk) InspectSecret(ctx context.Context, name string) error { - fmt.Printf("... done. \n") - return nil +func (s sdk) InspectSecret(ctx context.Context, id string) (docker.Secret, error) { + logrus.Debug("Inspect secret " + id) + response, err := s.SM.DescribeSecret(&secretsmanager.DescribeSecretInput{SecretId: &id}) + if err != nil { + return docker.Secret{}, err + } + labels := map[string]string{} + for _, tag := range response.Tags { + labels[*tag.Key] = *tag.Value + } + secret := docker.Secret{ + ID: *response.ARN, + Name: *response.Name, + Labels: labels, + } + if response.Description != nil { + secret.Description = *response.Description + } + return secret, nil } -func (s sdk) ListSecrets(ctx context.Context) error { - fmt.Printf("... done. \n") - return nil +func (s sdk) ListSecrets(ctx context.Context) ([]docker.Secret, error) { + + logrus.Debug("List secrets ...") + response, err := s.SM.ListSecrets(&secretsmanager.ListSecretsInput{}) + if err != nil { + return []docker.Secret{}, err + } + var secrets []docker.Secret + + for _, sec := range response.SecretList { + + labels := map[string]string{} + for _, tag := range sec.Tags { + labels[*tag.Key] = *tag.Value + } + description := "" + if sec.Description != nil { + description = *sec.Description + } + secrets = append(secrets, docker.Secret{ + ID: *sec.ARN, + Name: *sec.Name, + Labels: labels, + Description: description, + }) + } + return secrets, nil } -func (s sdk) DeleteSecret(ctx context.Context, name string) error { - fmt.Printf("... done. \n") - return nil +func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { + logrus.Debug("List secrets ...") + force := !recover + _, err := s.SM.DeleteSecret(&secretsmanager.DeleteSecretInput{SecretId: &id, ForceDeleteWithoutRecovery: &force}) + return err } diff --git a/ecs/pkg/amazon/secrets.go b/ecs/pkg/amazon/secrets.go index daabe79b..649705f0 100644 --- a/ecs/pkg/amazon/secrets.go +++ b/ecs/pkg/amazon/secrets.go @@ -2,27 +2,29 @@ package amazon import ( "context" + + "github.com/docker/ecs-plugin/pkg/docker" ) type secretsAPI interface { CreateSecret(ctx context.Context, name string, content string) (string, error) - InspectSecret(ctx context.Context, name string) error - ListSecrets(ctx context.Context) error - DeleteSecret(ctx context.Context, name string) error + InspectSecret(ctx context.Context, id string) (docker.Secret, error) + ListSecrets(ctx context.Context) ([]docker.Secret, error) + DeleteSecret(ctx context.Context, id string, recover bool) error } func (c client) CreateSecret(ctx context.Context, name string, content string) (string, error) { return c.api.CreateSecret(ctx, name, content) } -func (c client) InspectSecret(ctx context.Context, name string) error { - return c.api.InspectSecret(ctx, name) +func (c client) InspectSecret(ctx context.Context, id string) (docker.Secret, error) { + return c.api.InspectSecret(ctx, id) } -func (c client) ListSecrets(ctx context.Context) error { +func (c client) ListSecrets(ctx context.Context) ([]docker.Secret, error) { return c.api.ListSecrets(ctx) } -func (c client) DeleteSecret(ctx context.Context, name string) error { - return c.api.DeleteSecret(ctx, name) +func (c client) DeleteSecret(ctx context.Context, id string, recover bool) error { + return c.api.DeleteSecret(ctx, id, recover) } diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 32095102..e23651b6 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -4,6 +4,7 @@ import ( "context" "github.com/awslabs/goformation/v4/cloudformation" + "github.com/docker/ecs-plugin/pkg/docker" ) type API interface { @@ -11,8 +12,8 @@ type API interface { ComposeUp(ctx context.Context, project *Project) error ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error - CreateSecret(ctx context.Context, name string, content string) (string, error) - InspectSecret(ctx context.Context, name string) error - ListSecrets(ctx context.Context) error - DeleteSecret(ctx context.Context, name string) error + CreateSecret(ctx context.Context, name string, secret string) (string, error) + InspectSecret(ctx context.Context, id string) (docker.Secret, error) + ListSecrets(ctx context.Context) ([]docker.Secret, error) + DeleteSecret(ctx context.Context, id string, recover bool) error } diff --git a/ecs/pkg/docker/secret.go b/ecs/pkg/docker/secret.go new file mode 100644 index 00000000..0efae5d6 --- /dev/null +++ b/ecs/pkg/docker/secret.go @@ -0,0 +1,20 @@ +package docker + +import ( + "encoding/json" +) + +type Secret struct { + ID string `json:"ID"` + Name string `json:"Name"` + Labels map[string]string `json:"Labels"` + Description string `json:"Description"` +} + +func (s Secret) ToJSON() (string, error) { + b, err := json.MarshalIndent(&s, "", "\t") + if err != nil { + return "", err + } + return string(b), nil +} From 678f4018f077ea6e274c499995b268365fdd2c21 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 28 Apr 2020 16:48:17 +0200 Subject: [PATCH 042/205] Collect events while waiting for stack to complete Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/mock/api.go | 26 ++++++++++++++++----- ecs/pkg/amazon/sdk.go | 46 +++++++++++++++++++++++++++++++++----- ecs/pkg/amazon/up.go | 26 +++++++++++++++++++-- 3 files changed, 86 insertions(+), 12 deletions(-) diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index fd9b01c0..571f0bf2 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -6,7 +6,8 @@ package mock import ( context "context" - cloudformation "github.com/awslabs/goformation/v4/cloudformation" + cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" + cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" gomock "github.com/golang/mock/gomock" reflect "reflect" ) @@ -65,7 +66,7 @@ func (mr *MockAPIMockRecorder) CreateCluster(arg0, arg1 interface{}) *gomock.Cal } // CreateStack mocks base method -func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation.Template) error { +func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation0.Template) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2) ret0, _ := ret[0].(error) @@ -107,11 +108,12 @@ func (mr *MockAPIMockRecorder) DeleteStack(arg0, arg1 interface{}) *gomock.Call } // DescribeStackEvents mocks base method -func (m *MockAPI) DescribeStackEvents(arg0 context.Context, arg1 string) error { +func (m *MockAPI) DescribeStackEvents(arg0 context.Context, arg1 string) ([]*cloudformation.StackEvent, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DescribeStackEvents", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].([]*cloudformation.StackEvent) + ret1, _ := ret[1].(error) + return ret0, ret1 } // DescribeStackEvents indicates an expected call of DescribeStackEvents @@ -209,3 +211,17 @@ func (mr *MockAPIMockRecorder) VpcExists(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VpcExists", reflect.TypeOf((*MockAPI)(nil).VpcExists), arg0, arg1) } + +// WaitStackComplete mocks base method +func (m *MockAPI) WaitStackComplete(arg0 context.Context, arg1 string, arg2 func() error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitStackComplete", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// WaitStackComplete indicates an expected call of WaitStackComplete +func (mr *MockAPIMockRecorder) WaitStackComplete(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitStackComplete", reflect.TypeOf((*MockAPI)(nil).WaitStackComplete), arg0, arg1, arg2) +} diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 8f8c3403..95897511 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -3,6 +3,8 @@ package amazon import ( "context" "fmt" + "strings" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -183,13 +185,47 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template }) return err } +func (s sdk) WaitStackComplete(ctx context.Context, name string, fn func() error) error { + for i := 0; i < 120; i++ { + stacks, err := s.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ + StackName: aws.String(name), + }) + if err != nil { + return err + } -func (s sdk) DescribeStackEvents(ctx context.Context, name string) error { + err = fn() + if err != nil { + return err + } + + status := *stacks.Stacks[0].StackStatus + if strings.HasSuffix(status, "_COMPLETE") || strings.HasSuffix(status, "_FAILED") { + return nil + } + time.Sleep(1 * time.Second) + } + return fmt.Errorf("120s timeout waiting for CloudFormation stack %s to complete", name) +} + +func (s sdk) DescribeStackEvents(ctx context.Context, name string) ([]*cloudformation.StackEvent, error) { // Fixme implement Paginator on Events and return as a chan(events) - _, err := s.CF.DescribeStackEventsWithContext(aws.Context(ctx), &cloudformation.DescribeStackEventsInput{ - StackName: aws.String(name), - }) - return err + events := []*cloudformation.StackEvent{} + var nextToken *string + for { + resp, err := s.CF.DescribeStackEventsWithContext(aws.Context(ctx), &cloudformation.DescribeStackEventsInput{ + StackName: aws.String(name), + NextToken: nextToken, + }) + if err != nil { + return nil, err + } + events = append(events, resp.StackEvents...) + if resp.NextToken == nil { + return events, nil + } + nextToken = resp.NextToken + } } func (s sdk) DeleteStack(ctx context.Context, name string) error { diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 41b4f5cb..5b964b6e 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + cf "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/awslabs/goformation/v4/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -34,7 +36,26 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return err } - err = c.api.DescribeStackEvents(ctx, project.Name) + known := map[string]struct{}{} + err = c.api.WaitStackComplete(ctx, project.Name, func() error { + events, err := c.api.DescribeStackEvents(ctx, project.Name) + if err != nil { + return err + } + for _, event := range events { + if _, ok := known[*event.EventId]; ok { + continue + } + known[*event.EventId] = struct{}{} + + description := "-" + if event.ResourceStatusReason != nil { + description = *event.ResourceStatusReason + } + fmt.Printf("%s %q %s %s\n", *event.ResourceType, *event.LogicalResourceId, *event.ResourceStatus, description) + } + return nil + }) if err != nil { return err } @@ -48,5 +69,6 @@ type upAPI interface { CreateCluster(ctx context.Context, name string) (string, error) StackExists(ctx context.Context, name string) (bool, error) CreateStack(ctx context.Context, name string, template *cloudformation.Template) error - DescribeStackEvents(ctx context.Context, stack string) error + WaitStackComplete(ctx context.Context, name string, fn func() error) error + DescribeStackEvents(ctx context.Context, stack string) ([]*cf.StackEvent, error) } From 5d61fc119a5618b4bf5619b8d3ffd58929bf67f7 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 29 Apr 2020 15:08:26 +0200 Subject: [PATCH 043/205] Format stack events as a set of resources with progress status This mimic how docker-compose report containers creation Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/up.go | 17 ++-- ecs/pkg/console/progress.go | 134 +++++++++++++++++++++++++++++++ ecs/pkg/console/progress_test.go | 65 +++++++++++++++ 3 files changed, 210 insertions(+), 6 deletions(-) create mode 100644 ecs/pkg/console/progress.go create mode 100644 ecs/pkg/console/progress_test.go diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 5b964b6e..c7a50409 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -3,11 +3,13 @@ package amazon import ( "context" "fmt" + "sort" + "github.com/aws/aws-sdk-go/aws" cf "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/awslabs/goformation/v4/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/console" ) func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error { @@ -36,23 +38,26 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return err } + w := console.NewProgressWriter() known := map[string]struct{}{} err = c.api.WaitStackComplete(ctx, project.Name, func() error { events, err := c.api.DescribeStackEvents(ctx, project.Name) if err != nil { return err } + + sort.Slice(events, func(i, j int) bool { + return events[i].Timestamp.Before(*events[j].Timestamp) + }) + for _, event := range events { if _, ok := known[*event.EventId]; ok { continue } known[*event.EventId] = struct{}{} - description := "-" - if event.ResourceStatusReason != nil { - description = *event.ResourceStatusReason - } - fmt.Printf("%s %q %s %s\n", *event.ResourceType, *event.LogicalResourceId, *event.ResourceStatus, description) + resource := fmt.Sprintf("%s %q", aws.StringValue(event.ResourceType), aws.StringValue(event.LogicalResourceId)) + w.ResourceEvent(resource, aws.StringValue(event.ResourceStatus), aws.StringValue(event.ResourceStatusReason)) } return nil }) diff --git a/ecs/pkg/console/progress.go b/ecs/pkg/console/progress.go new file mode 100644 index 00000000..8ada4df8 --- /dev/null +++ b/ecs/pkg/console/progress.go @@ -0,0 +1,134 @@ +package console + +import ( + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/sirupsen/logrus" +) + +type resource struct { + name string + status string + details string +} + +type progress struct { + console console + resources []*resource +} + +func NewProgressWriter() *progress { + return &progress{ + console: ansiConsole{os.Stdout}, + } +} + +const ( + cyan = "36;1" + red = "31;1" + green = "32;1" +) + +func (p *progress) ResourceEvent(name string, status string, details string) { + if logrus.IsLevelEnabled(logrus.DebugLevel) { + logrus.Debugf("> %s : %s %s\n", name, status, details) + return + } + p.console.MoveUp(len(p.resources)) + + newResource := true + for _, r := range p.resources { + if r.name == name { + newResource = false + r.status = status + r.details = details + break + } + } + if newResource { + p.resources = append(p.resources, &resource{name, status, details}) + } + + var width int + for _, r := range p.resources { + l := len(r.name) + if width < l { + width = l + } + } + + for _, r := range p.resources { + s := r.status + if strings.HasSuffix(s, "_IN_PROGRESS") { + s = p.console.WiP(s) + } else if strings.HasSuffix(s, "_COMPLETE") { + s = p.console.OK(s) + } else if strings.HasSuffix(s, "_FAILED") { + s = p.console.KO(s) + } + p.console.ClearLine() + p.console.Printf("%-"+strconv.Itoa(width)+"s ... %s %s", r.name, s, r.details) // nolint:errcheck + p.console.MoveDown(1) + } +} + +type console interface { + Printf(format string, a ...interface{}) + MoveUp(int) + MoveDown(int) + ClearLine() + OK(string) string + KO(string) string + WiP(string) string +} + +type ansiConsole struct { + out io.Writer +} + +func (c ansiConsole) Printf(format string, a ...interface{}) { + fmt.Fprintf(c.out, format, a...) // nolint:errcheck + fmt.Fprintf(c.out, "\r") +} + +func (c ansiConsole) MoveUp(i int) { + if i == 0 { + return + } + fmt.Fprintf(c.out, "\033[%dA", i) // nolint:errcheck +} + +func (c ansiConsole) MoveDown(i int) { + if i == 0 { + return + } + fmt.Fprintf(c.out, "\033[%dB", i) // nolint:errcheck +} + +func (c ansiConsole) ClearLine() { + fmt.Fprint(c.out, "\033[2K\r") // nolint:errcheck +} + +func (c ansiConsole) OK(s string) string { + return ansiColor(green, s) +} + +func (c ansiConsole) KO(s string) string { + return ansiColor(red, s) +} + +func (c ansiConsole) WiP(s string) string { + return ansiColor(cyan, s) +} + +func ansiColor(code, s string) string { + return fmt.Sprintf("%s%s%s", ansi(code), s, ansi("0")) +} + +func ansi(code string) string { + return fmt.Sprintf("\033[%sm", code) +} diff --git a/ecs/pkg/console/progress_test.go b/ecs/pkg/console/progress_test.go new file mode 100644 index 00000000..552303c2 --- /dev/null +++ b/ecs/pkg/console/progress_test.go @@ -0,0 +1,65 @@ +package console + +import ( + "fmt" + "testing" + + "gotest.tools/v3/assert" +) + +func TestProgressWriter(t *testing.T) { + c := &bufferConsole{} + p := progress{ + console: c, + } + p.ResourceEvent("resource1", "CREATE_IN_PROGRESS", "") + assert.Equal(t, c.lines[0], "resource1 ... CREATE_IN_PROGRESS ") + + p.ResourceEvent("resource2_long_name", "CREATE_IN_PROGRESS", "ok") + assert.Equal(t, c.lines[0], "resource1 ... CREATE_IN_PROGRESS ") + assert.Equal(t, c.lines[1], "resource2_long_name ... CREATE_IN_PROGRESS ok") + + p.ResourceEvent("resource2_long_name", "CREATE_COMPLETE", "done") + assert.Equal(t, c.lines[0], "resource1 ... CREATE_IN_PROGRESS ") + assert.Equal(t, c.lines[1], "resource2_long_name ... CREATE_COMPLETE done") + + p.ResourceEvent("resource1", "CREATE_FAILED", "oups") + assert.Equal(t, c.lines[0], "resource1 ... CREATE_FAILED oups") + assert.Equal(t, c.lines[1], "resource2_long_name ... CREATE_COMPLETE done") +} + +type bufferConsole struct { + pos int + lines []string +} + +func (b *bufferConsole) Printf(format string, a ...interface{}) { + b.lines[b.pos] = fmt.Sprintf(format, a...) +} + +func (b *bufferConsole) MoveUp(i int) { + b.pos -= i +} + +func (b *bufferConsole) MoveDown(i int) { + b.pos += i +} + +func (b *bufferConsole) ClearLine() { + if len(b.lines) <= b.pos { + b.lines = append(b.lines, "") + } + b.lines[b.pos] = "" +} + +func (b *bufferConsole) OK(s string) string { + return s +} + +func (b *bufferConsole) KO(s string) string { + return s +} + +func (b *bufferConsole) WiP(s string) string { + return s +} From 814259ae33be3daf17a008194892519185e6b69a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 29 Apr 2020 15:26:11 +0200 Subject: [PATCH 044/205] Also wait for deletion events Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/down.go | 5 ++++ ecs/pkg/amazon/down_test.go | 2 ++ ecs/pkg/amazon/up.go | 37 ++--------------------------- ecs/pkg/amazon/wait.go | 46 +++++++++++++++++++++++++++++++++++++ ecs/pkg/console/progress.go | 6 ++++- 5 files changed, 60 insertions(+), 36 deletions(-) create mode 100644 ecs/pkg/amazon/wait.go diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index ab708d21..616d6659 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -12,6 +12,11 @@ func (c *client) ComposeDown(ctx context.Context, projectName string, deleteClus } fmt.Printf("Delete stack ") + err = c.WaitStackCompletion(ctx, projectName) + if err != nil { + return err + } + if !deleteCluster { return nil } diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go index 3bfc1e40..49e5fe2a 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/down_test.go @@ -20,6 +20,7 @@ func TestDownDontDeleteCluster(t *testing.T) { ctx := context.TODO() recorder := m.EXPECT() recorder.DeleteStack(ctx, "test_project").Return(nil).Times(1) + recorder.WaitStackComplete(ctx, "test_project", gomock.Any()).Return(nil).Times(1) c.ComposeDown(ctx, "test_project", false) } @@ -37,6 +38,7 @@ func TestDownDeleteCluster(t *testing.T) { ctx := context.TODO() recorder := m.EXPECT() recorder.DeleteStack(ctx, "test_project").Return(nil).Times(1) + recorder.WaitStackComplete(ctx, "test_project", gomock.Any()).Return(nil).Times(1) recorder.DeleteCluster(ctx, "test_cluster").Return(nil).Times(1) c.ComposeDown(ctx, "test_project", true) diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index c7a50409..0ee37e5f 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -3,13 +3,9 @@ package amazon import ( "context" "fmt" - "sort" - "github.com/aws/aws-sdk-go/aws" - cf "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/awslabs/goformation/v4/cloudformation" "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/pkg/console" ) func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error { @@ -38,42 +34,13 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return err } - w := console.NewProgressWriter() - known := map[string]struct{}{} - err = c.api.WaitStackComplete(ctx, project.Name, func() error { - events, err := c.api.DescribeStackEvents(ctx, project.Name) - if err != nil { - return err - } - - sort.Slice(events, func(i, j int) bool { - return events[i].Timestamp.Before(*events[j].Timestamp) - }) - - for _, event := range events { - if _, ok := known[*event.EventId]; ok { - continue - } - known[*event.EventId] = struct{}{} - - resource := fmt.Sprintf("%s %q", aws.StringValue(event.ResourceType), aws.StringValue(event.LogicalResourceId)) - w.ResourceEvent(resource, aws.StringValue(event.ResourceStatus), aws.StringValue(event.ResourceStatusReason)) - } - return nil - }) - if err != nil { - return err - } - - // TODO monitor progress - return nil + return c.WaitStackCompletion(ctx, project.Name) } type upAPI interface { + waitAPI ClusterExists(ctx context.Context, name string) (bool, error) CreateCluster(ctx context.Context, name string) (string, error) StackExists(ctx context.Context, name string) (bool, error) CreateStack(ctx context.Context, name string, template *cloudformation.Template) error - WaitStackComplete(ctx context.Context, name string, fn func() error) error - DescribeStackEvents(ctx context.Context, stack string) ([]*cf.StackEvent, error) } diff --git a/ecs/pkg/amazon/wait.go b/ecs/pkg/amazon/wait.go new file mode 100644 index 00000000..f3a25c89 --- /dev/null +++ b/ecs/pkg/amazon/wait.go @@ -0,0 +1,46 @@ +package amazon + +import ( + "context" + "fmt" + "sort" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/docker/ecs-plugin/pkg/console" +) + +func (c *client) WaitStackCompletion(ctx context.Context, name string) error { + w := console.NewProgressWriter() + known := map[string]struct{}{} + err := c.api.WaitStackComplete(ctx, name, func() error { + events, err := c.api.DescribeStackEvents(ctx, name) + if err != nil { + return err + } + + sort.Slice(events, func(i, j int) bool { + return events[i].Timestamp.Before(*events[j].Timestamp) + }) + + for _, event := range events { + if _, ok := known[*event.EventId]; ok { + continue + } + known[*event.EventId] = struct{}{} + + resource := fmt.Sprintf("%s %q", aws.StringValue(event.ResourceType), aws.StringValue(event.LogicalResourceId)) + w.ResourceEvent(resource, aws.StringValue(event.ResourceStatus), aws.StringValue(event.ResourceStatusReason)) + } + return nil + }) + if err != nil { + return err + } + return nil +} + +type waitAPI interface { + WaitStackComplete(ctx context.Context, name string, fn func() error) error + DescribeStackEvents(ctx context.Context, stack string) ([]*cloudformation.StackEvent, error) +} diff --git a/ecs/pkg/console/progress.go b/ecs/pkg/console/progress.go index 8ada4df8..157ee813 100644 --- a/ecs/pkg/console/progress.go +++ b/ecs/pkg/console/progress.go @@ -21,7 +21,11 @@ type progress struct { resources []*resource } -func NewProgressWriter() *progress { +type ProgressWriter interface { + ResourceEvent(name string, status string, details string) +} + +func NewProgressWriter() ProgressWriter { return &progress{ console: ansiConsole{os.Stdout}, } From f69ada632a46d3ff1f8160cc9c81359239f26e7b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 29 Apr 2020 15:35:34 +0200 Subject: [PATCH 045/205] use faint for "in progress" to distingish completed/failed Signed-off-by: Nicolas De Loof --- ecs/pkg/console/progress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/pkg/console/progress.go b/ecs/pkg/console/progress.go index 157ee813..6943dd8a 100644 --- a/ecs/pkg/console/progress.go +++ b/ecs/pkg/console/progress.go @@ -32,7 +32,7 @@ func NewProgressWriter() ProgressWriter { } const ( - cyan = "36;1" + blue = "36;2" red = "31;1" green = "32;1" ) @@ -126,7 +126,7 @@ func (c ansiConsole) KO(s string) string { } func (c ansiConsole) WiP(s string) string { - return ansiColor(cyan, s) + return ansiColor(blue, s) } func ansiColor(code, s string) string { From a8e963a304ab932a97fa71a3058356a94d97c995 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 30 Apr 2020 15:42:34 +0200 Subject: [PATCH 046/205] Query stack events by stack ID (not name) This prevent a race condition on `down` as stack is deleted and we still ask for stack events as we didn't recieved the DELETE_COMPLETE one Use WaitUntilStack* to detect stack operation completion Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/down.go | 3 +- ecs/pkg/amazon/mock/api.go | 17 ++++++++- ecs/pkg/amazon/sdk.go | 71 ++++++++++++++++++------------------- ecs/pkg/amazon/up.go | 2 +- ecs/pkg/amazon/wait.go | 53 ++++++++++++++++++++------- ecs/pkg/console/progress.go | 6 ---- ecs/pkg/convert/convert.go | 1 - 7 files changed, 93 insertions(+), 60 deletions(-) diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go index 616d6659..d9bee735 100644 --- a/ecs/pkg/amazon/down.go +++ b/ecs/pkg/amazon/down.go @@ -10,9 +10,8 @@ func (c *client) ComposeDown(ctx context.Context, projectName string, deleteClus if err != nil { return err } - fmt.Printf("Delete stack ") - err = c.WaitStackCompletion(ctx, projectName) + err = c.WaitStackCompletion(ctx, projectName, StackDelete) if err != nil { return err } diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index 571f0bf2..81b0829d 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -152,6 +152,21 @@ func (mr *MockAPIMockRecorder) GetRoleArn(arg0, arg1 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoleArn", reflect.TypeOf((*MockAPI)(nil).GetRoleArn), arg0, arg1) } +// GetStackID mocks base method +func (m *MockAPI) GetStackID(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStackID", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStackID indicates an expected call of GetStackID +func (mr *MockAPIMockRecorder) GetStackID(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStackID", reflect.TypeOf((*MockAPI)(nil).GetStackID), arg0, arg1) +} + // GetSubNets mocks base method func (m *MockAPI) GetSubNets(arg0 context.Context, arg1 string) ([]string, error) { m.ctrl.T.Helper() @@ -213,7 +228,7 @@ func (mr *MockAPIMockRecorder) VpcExists(arg0, arg1 interface{}) *gomock.Call { } // WaitStackComplete mocks base method -func (m *MockAPI) WaitStackComplete(arg0 context.Context, arg1 string, arg2 func() error) error { +func (m *MockAPI) WaitStackComplete(arg0 context.Context, arg1 string, arg2 int) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "WaitStackComplete", arg0, arg1, arg2) ret0, _ := ret[0].(error) diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 95897511..29ca7f7b 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -3,8 +3,6 @@ package amazon import ( "context" "fmt" - "strings" - "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -53,7 +51,7 @@ func NewAPI(sess *session.Session) API { func (s sdk) ClusterExists(ctx context.Context, name string) (bool, error) { logrus.Debug("Check if cluster was already created: ", name) - clusters, err := s.ECS.DescribeClustersWithContext(aws.Context(ctx), &ecs.DescribeClustersInput{ + clusters, err := s.ECS.DescribeClustersWithContext(ctx, &ecs.DescribeClustersInput{ Clusters: []*string{aws.String(name)}, }) if err != nil { @@ -64,7 +62,7 @@ func (s sdk) ClusterExists(ctx context.Context, name string) (bool, error) { func (s sdk) CreateCluster(ctx context.Context, name string) (string, error) { logrus.Debug("Create cluster ", name) - response, err := s.ECS.CreateClusterWithContext(aws.Context(ctx), &ecs.CreateClusterInput{ClusterName: aws.String(name)}) + response, err := s.ECS.CreateClusterWithContext(ctx, &ecs.CreateClusterInput{ClusterName: aws.String(name)}) if err != nil { return "", err } @@ -73,7 +71,7 @@ func (s sdk) CreateCluster(ctx context.Context, name string) (string, error) { func (s sdk) DeleteCluster(ctx context.Context, name string) error { logrus.Debug("Delete cluster ", name) - response, err := s.ECS.DeleteClusterWithContext(aws.Context(ctx), &ecs.DeleteClusterInput{Cluster: aws.String(name)}) + response, err := s.ECS.DeleteClusterWithContext(ctx, &ecs.DeleteClusterInput{Cluster: aws.String(name)}) if err != nil { return err } @@ -85,13 +83,13 @@ func (s sdk) DeleteCluster(ctx context.Context, name string) error { func (s sdk) VpcExists(ctx context.Context, vpcID string) (bool, error) { logrus.Debug("Check if VPC exists: ", vpcID) - _, err := s.EC2.DescribeVpcsWithContext(aws.Context(ctx), &ec2.DescribeVpcsInput{VpcIds: []*string{&vpcID}}) + _, err := s.EC2.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{VpcIds: []*string{&vpcID}}) return err == nil, err } func (s sdk) GetDefaultVPC(ctx context.Context) (string, error) { logrus.Debug("Retrieve default VPC") - vpcs, err := s.EC2.DescribeVpcsWithContext(aws.Context(ctx), &ec2.DescribeVpcsInput{ + vpcs, err := s.EC2.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{ Filters: []*ec2.Filter{ { Name: aws.String("isDefault"), @@ -110,7 +108,7 @@ func (s sdk) GetDefaultVPC(ctx context.Context) (string, error) { func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) { logrus.Debug("Retrieve SubNets") - subnets, err := s.EC2.DescribeSubnetsWithContext(aws.Context(ctx), &ec2.DescribeSubnetsInput{ + subnets, err := s.EC2.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{ DryRun: nil, Filters: []*ec2.Filter{ { @@ -135,7 +133,7 @@ func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) { } func (s sdk) ListRolesForPolicy(ctx context.Context, policy string) ([]string, error) { - entities, err := s.IAM.ListEntitiesForPolicyWithContext(aws.Context(ctx), &iam.ListEntitiesForPolicyInput{ + entities, err := s.IAM.ListEntitiesForPolicyWithContext(ctx, &iam.ListEntitiesForPolicyInput{ EntityFilter: aws.String("Role"), PolicyArn: aws.String(policy), }) @@ -150,7 +148,7 @@ func (s sdk) ListRolesForPolicy(ctx context.Context, policy string) ([]string, e } func (s sdk) GetRoleArn(ctx context.Context, name string) (string, error) { - role, err := s.IAM.GetRoleWithContext(aws.Context(ctx), &iam.GetRoleInput{ + role, err := s.IAM.GetRoleWithContext(ctx, &iam.GetRoleInput{ RoleName: aws.String(name), }) if err != nil { @@ -160,7 +158,7 @@ func (s sdk) GetRoleArn(ctx context.Context, name string) (string, error) { } func (s sdk) StackExists(ctx context.Context, name string) (bool, error) { - stacks, err := s.CF.DescribeStacksWithContext(aws.Context(ctx), &cloudformation.DescribeStacksInput{ + stacks, err := s.CF.DescribeStacksWithContext(ctx, &cloudformation.DescribeStacksInput{ StackName: aws.String(name), }) if err != nil { @@ -177,7 +175,7 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template return err } - _, err = s.CF.CreateStackWithContext(aws.Context(ctx), &cloudformation.CreateStackInput{ + _, err = s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{ OnFailure: aws.String("DELETE"), StackName: aws.String(name), TemplateBody: aws.String(string(json)), @@ -185,36 +183,37 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template }) return err } -func (s sdk) WaitStackComplete(ctx context.Context, name string, fn func() error) error { - for i := 0; i < 120; i++ { - stacks, err := s.CF.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: aws.String(name), - }) - if err != nil { - return err - } - - err = fn() - if err != nil { - return err - } - - status := *stacks.Stacks[0].StackStatus - if strings.HasSuffix(status, "_COMPLETE") || strings.HasSuffix(status, "_FAILED") { - return nil - } - time.Sleep(1 * time.Second) +func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) error { + input := &cloudformation.DescribeStacksInput{ + StackName: aws.String(name), + } + switch operation { + case StackCreate: + return s.CF.WaitUntilStackCreateCompleteWithContext(ctx, input) + case StackDelete: + return s.CF.WaitUntilStackDeleteCompleteWithContext(ctx, input) + default: + return fmt.Errorf("internal error: unexpected stack operation %d", operation) } - return fmt.Errorf("120s timeout waiting for CloudFormation stack %s to complete", name) } -func (s sdk) DescribeStackEvents(ctx context.Context, name string) ([]*cloudformation.StackEvent, error) { +func (s sdk) GetStackID(ctx context.Context, name string) (string, error) { + stacks, err := s.CF.DescribeStacksWithContext(ctx, &cloudformation.DescribeStacksInput{ + StackName: aws.String(name), + }) + if err != nil { + return "", err + } + return *stacks.Stacks[0].StackId, nil +} + +func (s sdk) DescribeStackEvents(ctx context.Context, stackID string) ([]*cloudformation.StackEvent, error) { // Fixme implement Paginator on Events and return as a chan(events) events := []*cloudformation.StackEvent{} var nextToken *string for { - resp, err := s.CF.DescribeStackEventsWithContext(aws.Context(ctx), &cloudformation.DescribeStackEventsInput{ - StackName: aws.String(name), + resp, err := s.CF.DescribeStackEventsWithContext(ctx, &cloudformation.DescribeStackEventsInput{ + StackName: aws.String(stackID), NextToken: nextToken, }) if err != nil { @@ -230,7 +229,7 @@ func (s sdk) DescribeStackEvents(ctx context.Context, name string) ([]*cloudform func (s sdk) DeleteStack(ctx context.Context, name string) error { logrus.Debug("Delete CloudFormation stack") - _, err := s.CF.DeleteStackWithContext(aws.Context(ctx), &cloudformation.DeleteStackInput{ + _, err := s.CF.DeleteStackWithContext(ctx, &cloudformation.DeleteStackInput{ StackName: aws.String(name), }) return err diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 0ee37e5f..459f95d5 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -34,7 +34,7 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return err } - return c.WaitStackCompletion(ctx, project.Name) + return c.WaitStackCompletion(ctx, project.Name, StackCreate) } type upAPI interface { diff --git a/ecs/pkg/amazon/wait.go b/ecs/pkg/amazon/wait.go index f3a25c89..095fcc6b 100644 --- a/ecs/pkg/amazon/wait.go +++ b/ecs/pkg/amazon/wait.go @@ -4,17 +4,42 @@ import ( "context" "fmt" "sort" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/docker/ecs-plugin/pkg/console" ) -func (c *client) WaitStackCompletion(ctx context.Context, name string) error { +func (c *client) WaitStackCompletion(ctx context.Context, name string, operation int) error { w := console.NewProgressWriter() - known := map[string]struct{}{} - err := c.api.WaitStackComplete(ctx, name, func() error { - events, err := c.api.DescribeStackEvents(ctx, name) + knownEvents := map[string]struct{}{} + + // Get the unique Stack ID so we can collect events without getting some from previous deployments with same name + stackID, err := c.api.GetStackID(ctx, name) + if err != nil { + return err + } + + ticker := time.NewTicker(1 * time.Second) + done := make(chan error) + + go func() { + err := c.api.WaitStackComplete(ctx, name, operation) + ticker.Stop() + done <- err + }() + + var completed bool + var waitErr error + for !completed { + select { + case err := <-done: + completed = true + waitErr = err + case <-ticker.C: + } + events, err := c.api.DescribeStackEvents(ctx, stackID) if err != nil { return err } @@ -24,23 +49,25 @@ func (c *client) WaitStackCompletion(ctx context.Context, name string) error { }) for _, event := range events { - if _, ok := known[*event.EventId]; ok { + if _, ok := knownEvents[*event.EventId]; ok { continue } - known[*event.EventId] = struct{}{} + knownEvents[*event.EventId] = struct{}{} resource := fmt.Sprintf("%s %q", aws.StringValue(event.ResourceType), aws.StringValue(event.LogicalResourceId)) w.ResourceEvent(resource, aws.StringValue(event.ResourceStatus), aws.StringValue(event.ResourceStatusReason)) } - return nil - }) - if err != nil { - return err } - return nil + return waitErr } type waitAPI interface { - WaitStackComplete(ctx context.Context, name string, fn func() error) error - DescribeStackEvents(ctx context.Context, stack string) ([]*cloudformation.StackEvent, error) + GetStackID(ctx context.Context, name string) (string, error) + WaitStackComplete(ctx context.Context, name string, operation int) error + DescribeStackEvents(ctx context.Context, stackID string) ([]*cloudformation.StackEvent, error) } + +const ( + StackCreate = iota + StackDelete +) diff --git a/ecs/pkg/console/progress.go b/ecs/pkg/console/progress.go index 6943dd8a..599ac5a7 100644 --- a/ecs/pkg/console/progress.go +++ b/ecs/pkg/console/progress.go @@ -100,16 +100,10 @@ func (c ansiConsole) Printf(format string, a ...interface{}) { } func (c ansiConsole) MoveUp(i int) { - if i == 0 { - return - } fmt.Fprintf(c.out, "\033[%dA", i) // nolint:errcheck } func (c ansiConsole) MoveDown(i int) { - if i == 0 { - return - } fmt.Fprintf(c.out, "\033[%dB", i) // nolint:errcheck } diff --git a/ecs/pkg/convert/convert.go b/ecs/pkg/convert/convert.go index 33e95c73..c7a8cab8 100644 --- a/ecs/pkg/convert/convert.go +++ b/ecs/pkg/convert/convert.go @@ -71,7 +71,6 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe Tags: nil, Volumes: []ecs.TaskDefinition_Volume{}, }, nil - } func toCPU(service types.ServiceConfig) string { From 99b4ed0bfdf320018c1a8d19163e027f5b7ac459 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 30 Apr 2020 16:14:36 +0200 Subject: [PATCH 047/205] Capture first failure to report root error to user on command completion Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/down_test.go | 14 +++++++++----- ecs/pkg/amazon/wait.go | 21 +++++++++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go index 49e5fe2a..bf98a6b9 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/down_test.go @@ -19,8 +19,10 @@ func TestDownDontDeleteCluster(t *testing.T) { } ctx := context.TODO() recorder := m.EXPECT() - recorder.DeleteStack(ctx, "test_project").Return(nil).Times(1) - recorder.WaitStackComplete(ctx, "test_project", gomock.Any()).Return(nil).Times(1) + recorder.DeleteStack(ctx, "test_project").Return(nil) + recorder.GetStackID(ctx, "test_project").Return("stack-123", nil) + recorder.WaitStackComplete(ctx, "stack-123", StackDelete).Return(nil) + recorder.DescribeStackEvents(ctx, "stack-123").Return(nil, nil) c.ComposeDown(ctx, "test_project", false) } @@ -37,9 +39,11 @@ func TestDownDeleteCluster(t *testing.T) { ctx := context.TODO() recorder := m.EXPECT() - recorder.DeleteStack(ctx, "test_project").Return(nil).Times(1) - recorder.WaitStackComplete(ctx, "test_project", gomock.Any()).Return(nil).Times(1) - recorder.DeleteCluster(ctx, "test_cluster").Return(nil).Times(1) + recorder.DeleteStack(ctx, "test_project").Return(nil) + recorder.GetStackID(ctx, "test_project").Return("stack-123", nil) + recorder.WaitStackComplete(ctx, "stack-123", StackDelete).Return(nil) + recorder.DescribeStackEvents(ctx, "stack-123").Return(nil, nil) + recorder.DeleteCluster(ctx, "test_cluster").Return(nil) c.ComposeDown(ctx, "test_project", true) } diff --git a/ecs/pkg/amazon/wait.go b/ecs/pkg/amazon/wait.go index 095fcc6b..58ae93d7 100644 --- a/ecs/pkg/amazon/wait.go +++ b/ecs/pkg/amazon/wait.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sort" + "strings" "time" "github.com/aws/aws-sdk-go/aws" @@ -22,21 +23,20 @@ func (c *client) WaitStackCompletion(ctx context.Context, name string, operation } ticker := time.NewTicker(1 * time.Second) - done := make(chan error) + done := make(chan bool) go func() { - err := c.api.WaitStackComplete(ctx, name, operation) + c.api.WaitStackComplete(ctx, stackID, operation) //nolint:errcheck ticker.Stop() - done <- err + done <- true }() var completed bool - var waitErr error + var stackErr error for !completed { select { - case err := <-done: + case <-done: completed = true - waitErr = err case <-ticker.C: } events, err := c.api.DescribeStackEvents(ctx, stackID) @@ -55,10 +55,15 @@ func (c *client) WaitStackCompletion(ctx context.Context, name string, operation knownEvents[*event.EventId] = struct{}{} resource := fmt.Sprintf("%s %q", aws.StringValue(event.ResourceType), aws.StringValue(event.LogicalResourceId)) - w.ResourceEvent(resource, aws.StringValue(event.ResourceStatus), aws.StringValue(event.ResourceStatusReason)) + reason := aws.StringValue(event.ResourceStatusReason) + status := aws.StringValue(event.ResourceStatus) + w.ResourceEvent(resource, status, reason) + if stackErr == nil && strings.HasSuffix(status, "_FAILED") { + stackErr = fmt.Errorf(reason) + } } } - return waitErr + return stackErr } type waitAPI interface { From 3e30f2cd1a3991d09c49f76b8ba5a0656cd3480f Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 4 May 2020 15:09:08 +0200 Subject: [PATCH 048/205] Create CloudWatch LogGroup and IAM TaskExecutionRole As part of the CloudFormation template, create a LogGroup and configure task with awslogs log-driver. Also create a dedicated IAM Role, with AmazonECSTaskExecutionRolePolicy. This one will later be fine-tuned to grant access to secrets/config and other AWS resources according to custom extensions close #42 Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 26 +++++++++++++----- ecs/pkg/amazon/iam.go | 31 ++++++++++++++++++++++ ecs/pkg/amazon/sdk.go | 3 +++ ecs/pkg/convert/convert.go | 45 +++++++++++++++++++------------- 4 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 ecs/pkg/amazon/iam.go diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index e5c4cd71..40bf8edd 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -6,15 +6,17 @@ import ( "fmt" "strings" - "github.com/compose-spec/compose-go/types" - "github.com/sirupsen/logrus" + "github.com/awslabs/goformation/v4/cloudformation/logs" ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" + "github.com/awslabs/goformation/v4/cloudformation/iam" + "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" "github.com/docker/ecs-plugin/pkg/convert" + "github.com/sirupsen/logrus" ) func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudformation.Template, error) { @@ -50,22 +52,32 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo VpcId: vpc, } + logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) + template.Resources["LogGroup"] = &logs.LogGroup{ + LogGroupName: logGroup, + } + for _, service := range project.Services { definition, err := convert.Convert(project, service) if err != nil { return nil, err } - role, err := c.GetEcsTaskExecutionRole(ctx, service) - if err != nil { - return nil, err + taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", service.Name) + template.Resources[taskExecutionRole] = &iam.Role{ + AssumeRolePolicyDocument: assumeRolePolicyDocument, + // Here we can grant access to secrets/configs using a Policy { Allow,ssm:GetParameters,secret|config ARN} + ManagedPolicyArns: []string{ + ECSTaskExecutionPolicy, + }, } - definition.TaskRoleArn = role + definition.ExecutionRoleArn = cloudformation.Ref(taskExecutionRole) + // FIXME definition.TaskRoleArn = ? taskDefinition := fmt.Sprintf("%sTaskDefinition", service.Name) template.Resources[taskDefinition] = definition - template.Resources[service.Name] = &ecs.Service{ + template.Resources[fmt.Sprintf("%sService", service.Name)] = &ecs.Service{ Cluster: c.Cluster, DesiredCount: 1, LaunchType: ecsapi.LaunchTypeFargate, diff --git a/ecs/pkg/amazon/iam.go b/ecs/pkg/amazon/iam.go new file mode 100644 index 00000000..38c4d833 --- /dev/null +++ b/ecs/pkg/amazon/iam.go @@ -0,0 +1,31 @@ +package amazon + +var assumeRolePolicyDocument = PolicyDocument{ + Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html + Statement: []PolicyStatement{ + { + Effect: "Allow", + Principal: PolicyPrincipal{ + Service: "ecs-tasks.amazonaws.com", + }, + Action: []string{"sts:AssumeRole"}, + }, + }, +} + +// could alternatively depend on https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/master/pkg/cloud/services/iam/types.go#L52 +type PolicyDocument struct { + Version string `json:",omitempty"` + Statement []PolicyStatement `json:",omitempty"` +} + +type PolicyStatement struct { + Effect string `json:",omitempty"` + Action []string `json:",omitempty"` + Principal PolicyPrincipal `json:",omitempty"` + Resource []string `json:",omitempty"` +} + +type PolicyPrincipal struct { + Service string `json:",omitempty"` +} diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 29ca7f7b..89d3ec31 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -180,6 +180,9 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template StackName: aws.String(name), TemplateBody: aws.String(string(json)), TimeoutInMinutes: aws.Int64(10), + Capabilities: []*string{ + aws.String(cloudformation.CapabilityCapabilityIam), + }, }) return err } diff --git a/ecs/pkg/convert/convert.go b/ecs/pkg/convert/convert.go index c7a8cab8..8b70a648 100644 --- a/ecs/pkg/convert/convert.go +++ b/ecs/pkg/convert/convert.go @@ -5,6 +5,7 @@ import ( "time" ecsapi "github.com/aws/aws-sdk-go/service/ecs" + "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/compose-spec/compose-go/types" "github.com/docker/cli/opts" @@ -21,24 +22,32 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe ContainerDefinitions: []ecs.TaskDefinition_ContainerDefinition{ // Here we can declare sidecars and init-containers using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_dependson { - Command: service.Command, - Cpu: 256, - DisableNetworking: service.NetworkMode == "none", - DnsSearchDomains: service.DNSSearch, - DnsServers: service.DNS, - DockerLabels: nil, - DockerSecurityOptions: service.SecurityOpt, - EntryPoint: service.Entrypoint, - Environment: toKeyValuePair(service.Environment), - Essential: true, - ExtraHosts: toHostEntryPtr(service.ExtraHosts), - FirelensConfiguration: nil, - HealthCheck: toHealthCheck(service.HealthCheck), - Hostname: service.Hostname, - Image: service.Image, - Interactive: false, - Links: nil, - LinuxParameters: toLinuxParameters(service), + Command: service.Command, + Cpu: 256, + DisableNetworking: service.NetworkMode == "none", + DnsSearchDomains: service.DNSSearch, + DnsServers: service.DNS, + DockerLabels: nil, + DockerSecurityOptions: service.SecurityOpt, + EntryPoint: service.Entrypoint, + Environment: toKeyValuePair(service.Environment), + Essential: true, + ExtraHosts: toHostEntryPtr(service.ExtraHosts), + FirelensConfiguration: nil, + HealthCheck: toHealthCheck(service.HealthCheck), + Hostname: service.Hostname, + Image: service.Image, + Interactive: false, + Links: nil, + LinuxParameters: toLinuxParameters(service), + LogConfiguration: &ecs.TaskDefinition_LogConfiguration{ + LogDriver: ecsapi.LogDriverAwslogs, + Options: map[string]string{ + "awslogs-region": cloudformation.Ref("AWS::Region"), + "awslogs-group": cloudformation.Ref("LogGroup"), + "awslogs-stream-prefix": service.Name, + }, + }, Memory: toMemoryLimits(service.Deploy), MemoryReservation: toMemoryReservation(service.Deploy), MountPoints: nil, From 2544307f55c5b0f3a0da8479f5836eaef0a49b3b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 4 May 2020 15:15:22 +0200 Subject: [PATCH 049/205] drop GetEcsTaskExecutionRole which is not in used anymore We need to define a way for compose-user to declare additional Policies to be added to TaskExecutionRole Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 36 ------------------- ecs/pkg/amazon/iam.go | 2 ++ ecs/pkg/amazon/mock/api.go | 59 ++++++++++++++++++++++++++++---- ecs/pkg/amazon/sdk.go | 15 -------- 4 files changed, 54 insertions(+), 58 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 40bf8edd..c805e099 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -13,10 +13,8 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/iam" - "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" "github.com/docker/ecs-plugin/pkg/convert" - "github.com/sirupsen/logrus" ) func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudformation.Template, error) { @@ -118,43 +116,9 @@ func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, e return defaultVPC, nil } -const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" - -var defaultTaskExecutionRole string - -// GetEcsTaskExecutionRole retrieve the role ARN to apply for task execution -func (c client) GetEcsTaskExecutionRole(ctx context.Context, spec types.ServiceConfig) (string, error) { - if arn, ok := spec.Extras["x-ecs-TaskExecutionRole"]; ok { - return arn.(string), nil - } - if defaultTaskExecutionRole != "" { - return defaultTaskExecutionRole, nil - } - - logrus.Debug("Retrieve Task Execution Role") - entities, err := c.api.ListRolesForPolicy(ctx, ECSTaskExecutionPolicy) - if err != nil { - return "", err - } - if len(entities) == 0 { - return "", fmt.Errorf("no Role is attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") - } - if len(entities) > 1 { - return "", fmt.Errorf("multiple Roles are attached to AmazonECSTaskExecutionRole Policy, please provide an explicit task execution role") - } - - arn, err := c.api.GetRoleArn(ctx, entities[0]) - if err != nil { - return "", err - } - defaultTaskExecutionRole = arn - return arn, nil -} - type convertAPI interface { GetDefaultVPC(ctx context.Context) (string, error) VpcExists(ctx context.Context, vpcID string) (bool, error) GetSubNets(ctx context.Context, vpcID string) ([]string, error) - ListRolesForPolicy(ctx context.Context, policy string) ([]string, error) GetRoleArn(ctx context.Context, name string) (string, error) } diff --git a/ecs/pkg/amazon/iam.go b/ecs/pkg/amazon/iam.go index 38c4d833..c07e34fe 100644 --- a/ecs/pkg/amazon/iam.go +++ b/ecs/pkg/amazon/iam.go @@ -1,5 +1,7 @@ package amazon +const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + var assumeRolePolicyDocument = PolicyDocument{ Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html Statement: []PolicyStatement{ diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index 81b0829d..4a116331 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -8,6 +8,7 @@ import ( context "context" cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" + docker "github.com/docker/ecs-plugin/pkg/docker" gomock "github.com/golang/mock/gomock" reflect "reflect" ) @@ -65,6 +66,21 @@ func (mr *MockAPIMockRecorder) CreateCluster(arg0, arg1 interface{}) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCluster", reflect.TypeOf((*MockAPI)(nil).CreateCluster), arg0, arg1) } +// CreateSecret mocks base method +func (m *MockAPI) CreateSecret(arg0 context.Context, arg1, arg2 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSecret", arg0, arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateSecret indicates an expected call of CreateSecret +func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSecret", reflect.TypeOf((*MockAPI)(nil).CreateSecret), arg0, arg1, arg2) +} + // CreateStack mocks base method func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation0.Template) error { m.ctrl.T.Helper() @@ -93,6 +109,20 @@ func (mr *MockAPIMockRecorder) DeleteCluster(arg0, arg1 interface{}) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockAPI)(nil).DeleteCluster), arg0, arg1) } +// DeleteSecret mocks base method +func (m *MockAPI) DeleteSecret(arg0 context.Context, arg1 string, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSecret", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteSecret indicates an expected call of DeleteSecret +func (mr *MockAPIMockRecorder) DeleteSecret(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSecret", reflect.TypeOf((*MockAPI)(nil).DeleteSecret), arg0, arg1, arg2) +} + // DeleteStack mocks base method func (m *MockAPI) DeleteStack(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() @@ -182,19 +212,34 @@ func (mr *MockAPIMockRecorder) GetSubNets(arg0, arg1 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), arg0, arg1) } -// ListRolesForPolicy mocks base method -func (m *MockAPI) ListRolesForPolicy(arg0 context.Context, arg1 string) ([]string, error) { +// InspectSecret mocks base method +func (m *MockAPI) InspectSecret(arg0 context.Context, arg1 string) (docker.Secret, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListRolesForPolicy", arg0, arg1) - ret0, _ := ret[0].([]string) + ret := m.ctrl.Call(m, "InspectSecret", arg0, arg1) + ret0, _ := ret[0].(docker.Secret) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListRolesForPolicy indicates an expected call of ListRolesForPolicy -func (mr *MockAPIMockRecorder) ListRolesForPolicy(arg0, arg1 interface{}) *gomock.Call { +// InspectSecret indicates an expected call of InspectSecret +func (mr *MockAPIMockRecorder) InspectSecret(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRolesForPolicy", reflect.TypeOf((*MockAPI)(nil).ListRolesForPolicy), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InspectSecret", reflect.TypeOf((*MockAPI)(nil).InspectSecret), arg0, arg1) +} + +// ListSecrets mocks base method +func (m *MockAPI) ListSecrets(arg0 context.Context) ([]docker.Secret, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListSecrets", arg0) + ret0, _ := ret[0].([]docker.Secret) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSecrets indicates an expected call of ListSecrets +func (mr *MockAPIMockRecorder) ListSecrets(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSecrets", reflect.TypeOf((*MockAPI)(nil).ListSecrets), arg0) } // StackExists mocks base method diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 89d3ec31..74cb0448 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -132,21 +132,6 @@ func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) { return ids, nil } -func (s sdk) ListRolesForPolicy(ctx context.Context, policy string) ([]string, error) { - entities, err := s.IAM.ListEntitiesForPolicyWithContext(ctx, &iam.ListEntitiesForPolicyInput{ - EntityFilter: aws.String("Role"), - PolicyArn: aws.String(policy), - }) - if err != nil { - return nil, err - } - roles := []string{} - for _, e := range entities.PolicyRoles { - roles = append(roles, *e.RoleName) - } - return roles, nil -} - func (s sdk) GetRoleArn(ctx context.Context, name string) (string, error) { role, err := s.IAM.GetRoleWithContext(ctx, &iam.GetRoleInput{ RoleName: aws.String(name), From 9a6fe86a86bab70f2f051236230cef6662c045c1 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 5 May 2020 13:50:57 +0200 Subject: [PATCH 050/205] Introduce "Validate" phase to check/make app ECS-compliant Signed-off-by: Nicolas De Loof --- ecs/README.md | 36 ++++++++++++++++---------- ecs/pkg/amazon/cloudformation.go | 3 +-- ecs/pkg/{convert => amazon}/convert.go | 20 +------------- ecs/pkg/amazon/up.go | 5 ++++ ecs/pkg/amazon/validate.go | 20 ++++++++++++++ 5 files changed, 50 insertions(+), 34 deletions(-) rename ecs/pkg/{convert => amazon}/convert.go (97%) create mode 100644 ecs/pkg/amazon/validate.go diff --git a/ecs/README.md b/ecs/README.md index 9ec4ab69..f15fd25f 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -11,21 +11,31 @@ template, which will create all resources in dependent order and cleanup on `down` command or deployment failure. ``` - +-----------------------------+ - | compose.yaml file | - +-----------------------------+ + +--------------------------------------+ + | compose.yaml file | + +--------------------------------------+ - Load - +-----------------------------+ - | compose-go Model | - +-----------------------------+ + +--------------------------------------+ + | compose Model | + +--------------------------------------+ +- Validate + +--------------------------------------+ + | compose Model suitable for ECS | + +--------------------------------------+ - Convert - +-----------------------------+ - | CloudFormation Template | - +-----------------------------+ + +--------------------------------------+ + | CloudFormation Template | + +--------------------------------------+ - Apply - +---------+ +------------+ - | AWS API | or | stack file | - +---------+ +------------+ + +--------------+ +----------------+ + | AWS API | or | stack file | + +--------------+ +----------------+ ``` -(if this sounds familiar, see [Kompose](https://github.com/kubernetes/kompose/blob/master/docs/architecture.md)) \ No newline at end of file +* _Load_ phase relies on [compose-go](https://github.com/compose-spec/compose-go). Any generic code we write for this +purpose should be proposed upstream. +* _Validate_ phase is responsible to inject sane ECS defaults into the compose-go model, and validate the `compose.yaml` +file do not include unsupported features. +* _Convert_ produces a CloudFormation template to define all resources required to implement the application model on AWS. +* _Apply_ phase do apply the CloudFormation template, either by exporting to a stack file or to deploy on AWS. + diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index c805e099..76aa8088 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -14,7 +14,6 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/pkg/convert" ) func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudformation.Template, error) { @@ -56,7 +55,7 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo } for _, service := range project.Services { - definition, err := convert.Convert(project, service) + definition, err := Convert(project, service) if err != nil { return nil, err } diff --git a/ecs/pkg/convert/convert.go b/ecs/pkg/amazon/convert.go similarity index 97% rename from ecs/pkg/convert/convert.go rename to ecs/pkg/amazon/convert.go index 8b70a648..a0bd9e42 100644 --- a/ecs/pkg/convert/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -1,4 +1,4 @@ -package convert +package amazon import ( "strings" @@ -174,16 +174,6 @@ func toUlimits(ulimits map[string]*types.UlimitsConfig) []ecs.TaskDefinition_Uli return u } -func uint32Toint64Ptr(i uint32) *int64 { - v := int64(i) - return &v -} - -func intToInt64Ptr(i int) *int64 { - v := int64(i) - return &v -} - const Mb = 1024 * 1024 func toMemoryLimits(deploy *types.DeployConfig) int { @@ -265,14 +255,6 @@ func toHealthCheck(check *types.HealthCheckConfig) *ecs.TaskDefinition_HealthChe } } -func uint64ToInt64Ptr(i *uint64) *int64 { - if i == nil { - return nil - } - v := int64(*i) - return &v -} - func durationToInt(interval *types.Duration) int { if interval == nil { return 0 diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 459f95d5..21adf973 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -24,6 +24,11 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") } + err = c.Validate(project) + if err != nil { + return err + } + template, err := c.Convert(ctx, project) if err != nil { return err diff --git a/ecs/pkg/amazon/validate.go b/ecs/pkg/amazon/validate.go new file mode 100644 index 00000000..5e61a273 --- /dev/null +++ b/ecs/pkg/amazon/validate.go @@ -0,0 +1,20 @@ +package amazon + +import ( + "github.com/compose-spec/compose-go/types" + "github.com/docker/ecs-plugin/pkg/compose" +) + +// Validate check the compose model do not use unsupported features and inject sane defaults for ECS deployment +func (c *client) Validate(project *compose.Project) error { + if len(project.Networks) == 0 { + // Compose application model implies a default network if none is explicitly set. + // FIXME move this to compose-go + project.Networks["default"] = types.NetworkConfig{ + Name: "default", + } + } + + // Here we can check for incompatible attributes, inject sane defaults, etc + return nil +} From 0eab586106e53d35677fd47d30c389b93200e4e7 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 5 May 2020 00:58:20 +0200 Subject: [PATCH 051/205] Create CloudMap private namespace and register services Signed-off-by: Nicolas De Loof --- ecs/Makefile | 2 +- ecs/cmd/{ => main}/main.go | 0 ecs/pkg/amazon/cloudformation.go | 21 +++++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) rename ecs/cmd/{ => main}/main.go (100%) diff --git a/ecs/Makefile b/ecs/Makefile index e07af815..5d3c5883 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -2,7 +2,7 @@ clean: rm -rf dist/ build: - go build -v -o dist/docker-ecs cmd/main.go + go build -v -o dist/docker-ecs cmd/main/main.go test: ## Run tests go test ./... -v diff --git a/ecs/cmd/main.go b/ecs/cmd/main/main.go similarity index 100% rename from ecs/cmd/main.go rename to ecs/cmd/main/main.go diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 76aa8088..7664d1a9 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -13,6 +13,7 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/iam" + cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -54,6 +55,13 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo LogGroupName: logGroup, } + // Private DNS namespace will allow DNS name for the services to be ..local + template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ + Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), + Name: fmt.Sprintf("%s.local", project.Name), + Vpc: vpc, + } + for _, service := range project.Services { definition, err := Convert(project, service) if err != nil { @@ -89,6 +97,19 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo ServiceName: service.Name, TaskDefinition: cloudformation.Ref(taskDefinition), } + + var healthCheck *cloudmap.Service_HealthCheckConfig + if service.HealthCheck != nil && !service.HealthCheck.Disable { + // FIXME ECS only support HTTP(s) health checks, while Docker only support CMD + } + + serviceRegistration := fmt.Sprintf("%sServiceRegistration", service.Name) + template.Resources[serviceRegistration] = &cloudmap.Service{ + Description: fmt.Sprintf("%q registration in Service Map", service.Name), + HealthCheckConfig: healthCheck, + Name: serviceRegistration, + NamespaceId: cloudformation.Ref("CloudMap"), + } } return template, nil } From 09871400efde3da8613a9b00a3efd9cbeed05c7a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 5 May 2020 12:06:32 +0200 Subject: [PATCH 052/205] Register services within Cloud Map close #35 Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 46 +++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 7664d1a9..4a9fcda9 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -6,13 +6,13 @@ import ( "fmt" "strings" - "github.com/awslabs/goformation/v4/cloudformation/logs" - ecsapi "github.com/aws/aws-sdk-go/service/ecs" + cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/iam" + "github.com/awslabs/goformation/v4/cloudformation/logs" cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -82,6 +82,28 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo taskDefinition := fmt.Sprintf("%sTaskDefinition", service.Name) template.Resources[taskDefinition] = definition + var healthCheck *cloudmap.Service_HealthCheckConfig + if service.HealthCheck != nil && !service.HealthCheck.Disable { + // FIXME ECS only support HTTP(s) health checks, while Docker only support CMD + } + + serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", service.Name) + template.Resources[serviceRegistration] = &cloudmap.Service{ + Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), + HealthCheckConfig: healthCheck, + Name: service.Name, + NamespaceId: cloudformation.Ref("CloudMap"), + DnsConfig: &cloudmap.Service_DnsConfig{ + DnsRecords: []cloudmap.Service_DnsRecord{ + { + TTL: 300, + Type: cloudmapapi.RecordTypeA, + }, + }, + RoutingPolicy: cloudmapapi.RoutingPolicyMultivalue, + }, + } + template.Resources[fmt.Sprintf("%sService", service.Name)] = &ecs.Service{ Cluster: c.Cluster, DesiredCount: 1, @@ -95,20 +117,12 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo }, SchedulingStrategy: ecsapi.SchedulingStrategyReplica, ServiceName: service.Name, - TaskDefinition: cloudformation.Ref(taskDefinition), - } - - var healthCheck *cloudmap.Service_HealthCheckConfig - if service.HealthCheck != nil && !service.HealthCheck.Disable { - // FIXME ECS only support HTTP(s) health checks, while Docker only support CMD - } - - serviceRegistration := fmt.Sprintf("%sServiceRegistration", service.Name) - template.Resources[serviceRegistration] = &cloudmap.Service{ - Description: fmt.Sprintf("%q registration in Service Map", service.Name), - HealthCheckConfig: healthCheck, - Name: serviceRegistration, - NamespaceId: cloudformation.Ref("CloudMap"), + ServiceRegistries: []ecs.Service_ServiceRegistry{ + { + RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), + }, + }, + TaskDefinition: cloudformation.Ref(taskDefinition), } } return template, nil From aa8587095f4b4e23d494b4b3f7997e0528eb8e64 Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Mon, 4 May 2020 18:35:23 +0200 Subject: [PATCH 053/205] Add setup command to define a docker context for ecs-plugin Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 15 +++----- ecs/cmd/commands/secret.go | 10 +++--- ecs/cmd/commands/setup.go | 33 ++++++++++++++++++ ecs/cmd/main/main.go | 21 ++++++++---- ecs/go.mod | 1 + ecs/pkg/docker/contextStore.go | 63 ++++++++++++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 ecs/cmd/commands/setup.go create mode 100644 ecs/pkg/docker/contextStore.go diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 972477b3..459a4881 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -6,16 +6,11 @@ import ( "github.com/docker/ecs-plugin/pkg/amazon" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" ) -type ClusterOptions struct { - Profile string - Region string - Cluster string -} - -func ComposeCommand(clusteropts *ClusterOptions) *cobra.Command { +func ComposeCommand(clusteropts *docker.AwsContext) *cobra.Command { cmd := &cobra.Command{ Use: "compose", } @@ -41,7 +36,7 @@ func (o upOptions) LoadBalancerArn() *string { return &o.loadBalancerArn } -func ConvertCommand(clusteropts *ClusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { +func ConvertCommand(clusteropts *docker.AwsContext, projectOpts *compose.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "convert", RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { @@ -66,7 +61,7 @@ func ConvertCommand(clusteropts *ClusterOptions, projectOpts *compose.ProjectOpt return cmd } -func UpCommand(clusteropts *ClusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { +func UpCommand(clusteropts *docker.AwsContext, projectOpts *compose.ProjectOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ Use: "up", @@ -86,7 +81,7 @@ type downOptions struct { DeleteCluster bool } -func DownCommand(clusteropts *ClusterOptions, projectOpts *compose.ProjectOptions) *cobra.Command { +func DownCommand(clusteropts *docker.AwsContext, projectOpts *compose.ProjectOptions) *cobra.Command { opts := downOptions{} cmd := &cobra.Command{ Use: "down", diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go index 0c228c54..b46eeeb3 100644 --- a/ecs/cmd/commands/secret.go +++ b/ecs/cmd/commands/secret.go @@ -22,7 +22,7 @@ type deleteSecretOptions struct { recover bool } -func SecretCommand(clusteropts *ClusterOptions) *cobra.Command { +func SecretCommand(clusteropts *docker.AwsContext) *cobra.Command { cmd := &cobra.Command{ Use: "secret", Short: "Manages secrets", @@ -37,7 +37,7 @@ func SecretCommand(clusteropts *ClusterOptions) *cobra.Command { return cmd } -func CreateSecret(clusteropts *ClusterOptions) *cobra.Command { +func CreateSecret(clusteropts *docker.AwsContext) *cobra.Command { //opts := createSecretOptions{} cmd := &cobra.Command{ Use: "create NAME SECRET", @@ -60,7 +60,7 @@ func CreateSecret(clusteropts *ClusterOptions) *cobra.Command { return cmd } -func InspectSecret(clusteropts *ClusterOptions) *cobra.Command { +func InspectSecret(clusteropts *docker.AwsContext) *cobra.Command { cmd := &cobra.Command{ Use: "inspect ID", Short: "Displays secret details", @@ -88,7 +88,7 @@ func InspectSecret(clusteropts *ClusterOptions) *cobra.Command { return cmd } -func ListSecrets(clusteropts *ClusterOptions) *cobra.Command { +func ListSecrets(clusteropts *docker.AwsContext) *cobra.Command { cmd := &cobra.Command{ Use: "list", Aliases: []string{"ls"}, @@ -110,7 +110,7 @@ func ListSecrets(clusteropts *ClusterOptions) *cobra.Command { return cmd } -func DeleteSecret(clusteropts *ClusterOptions) *cobra.Command { +func DeleteSecret(clusteropts *docker.AwsContext) *cobra.Command { opts := deleteSecretOptions{} cmd := &cobra.Command{ Use: "delete NAME", diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go new file mode 100644 index 00000000..a927605d --- /dev/null +++ b/ecs/cmd/commands/setup.go @@ -0,0 +1,33 @@ +package commands + +import ( + "github.com/docker/cli/cli-plugins/plugin" + contextStore "github.com/docker/ecs-plugin/pkg/docker" + "github.com/spf13/cobra" +) + +func SetupCommand() *cobra.Command { + var opts contextStore.AwsContext + var name string + cmd := &cobra.Command{ + Use: "setup", + Short: "", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + //Override the root command PersistentPreRun + //We just need to initialize the top parent command + return plugin.PersistentPreRunE(cmd, args) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return contextStore.NewContext(name, &opts) + }, + } + cmd.Flags().StringVarP(&name, "name", "n", "aws", "Context Name") + cmd.Flags().StringVarP(&opts.Profile, "profile", "p", "", "AWS Profile") + cmd.Flags().StringVarP(&opts.Cluster, "cluster", "c", "", "ECS cluster") + cmd.Flags().StringVarP(&opts.Region, "region", "r", "", "AWS region") + + cmd.MarkFlagRequired("profile") + cmd.MarkFlagRequired("cluster") + cmd.MarkFlagRequired("region") + return cmd +} diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 72d072a0..be8d2182 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -7,6 +7,7 @@ import ( "github.com/docker/cli/cli-plugins/plugin" "github.com/docker/cli/cli/command" commands "github.com/docker/ecs-plugin/cmd/commands" + "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" ) @@ -26,23 +27,29 @@ func main() { // NewRootCmd returns the base root command. func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { - var opts commands.ClusterOptions + var opts *docker.AwsContext cmd := &cobra.Command{ Short: "Docker ECS", Long: `run multi-container applications on Amazon ECS.`, Use: name, Annotations: map[string]string{"experimentalCLI": "true"}, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + err := plugin.PersistentPreRunE(cmd, args) + if err != nil { + return err + } + contextName := dockerCli.CurrentContext() + opts, err = docker.CheckAwsContextExists(contextName) + return err + }, } cmd.AddCommand( VersionCommand(), - commands.ComposeCommand(&opts), - commands.SecretCommand(&opts), + commands.ComposeCommand(opts), + commands.SecretCommand(opts), + commands.SetupCommand(), ) - cmd.Flags().StringVarP(&opts.Profile, "profile", "p", "default", "AWS Profile") - cmd.Flags().StringVarP(&opts.Cluster, "cluster", "c", "default", "ECS cluster") - cmd.Flags().StringVarP(&opts.Region, "region", "r", "", "AWS region") - return cmd } diff --git a/ecs/go.mod b/ecs/go.mod index 0ddf6c85..56489033 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -35,6 +35,7 @@ require ( github.com/lib/pq v1.3.0 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/miekg/pkcs11 v1.0.3 // indirect + github.com/mitchellh/mapstructure v1.2.2 github.com/morikuni/aec v1.0.0 // indirect github.com/onsi/ginkgo v1.11.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect diff --git a/ecs/pkg/docker/contextStore.go b/ecs/pkg/docker/contextStore.go new file mode 100644 index 00000000..2d40b9dd --- /dev/null +++ b/ecs/pkg/docker/contextStore.go @@ -0,0 +1,63 @@ +package docker + +import ( + "fmt" + + cliconfig "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/context/store" + "github.com/mitchellh/mapstructure" +) + +const contextType = "aws" + +type TypeContext struct { + Type string +} + +func getter() interface{} { + return &TypeContext{} +} + +type AwsContext struct { + Profile string + Cluster string + Region string +} + +func NewContext(name string, awsContext *AwsContext) error { + contextStore := initContextStore() + endpoints := map[string]interface{}{ + "aws": awsContext, + "docker": awsContext, + } + + metadata := store.Metadata{ + Name: name, + Endpoints: endpoints, + Metadata: TypeContext{Type: contextType}, + } + return contextStore.CreateOrUpdate(metadata) +} + +func initContextStore() store.Store { + config := store.NewConfig(getter) + return store.New(cliconfig.ContextStoreDir(), config) +} + +func CheckAwsContextExists(contextName string) (*AwsContext, error) { + contextStore := initContextStore() + metadata, err := contextStore.GetMetadata(contextName) + if err != nil { + return nil, err + } + endpoint := metadata.Endpoints["aws"] + awsContext := AwsContext{} + err = mapstructure.Decode(endpoint, &awsContext) + if err != nil { + return nil, err + } + if awsContext == (AwsContext{}) { + return nil, fmt.Errorf("can't use \"%s\" with ECS command because it is not an AWS context", contextName) + } + return &awsContext, nil +} From 6febf6874885c0b67c3aa0249c13f795b13535dd Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Tue, 5 May 2020 23:33:54 +0200 Subject: [PATCH 054/205] Add e2e tests for plugin behavior and commands Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/Makefile | 2 +- ecs/cmd/main/main.go | 6 + ecs/go.mod | 1 + ecs/pkg/docker/contextStore.go | 19 +++- ecs/tests/command_test.go | 18 +++ ecs/tests/main_test.go | 127 ++++++++++++++++++++++ ecs/tests/plugin_test.go | 33 ++++++ ecs/tests/setup_command_test.go | 34 ++++++ ecs/tests/testdata/context-inspect.golden | 16 +++ ecs/tests/testdata/plugin-usage.golden | 14 +++ 10 files changed, 264 insertions(+), 6 deletions(-) create mode 100644 ecs/tests/command_test.go create mode 100644 ecs/tests/main_test.go create mode 100644 ecs/tests/plugin_test.go create mode 100644 ecs/tests/setup_command_test.go create mode 100644 ecs/tests/testdata/context-inspect.golden create mode 100644 ecs/tests/testdata/plugin-usage.golden diff --git a/ecs/Makefile b/ecs/Makefile index 5d3c5883..126d5bc5 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -4,7 +4,7 @@ clean: build: go build -v -o dist/docker-ecs cmd/main/main.go -test: ## Run tests +test: build ## Run tests go test ./... -v dev: build diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index be8d2182..812ccfee 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -43,6 +43,12 @@ func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { opts, err = docker.CheckAwsContextExists(contextName) return err }, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return fmt.Errorf("%q is not a docker ecs command\nSee 'docker ecs --help'", args[0]) + } + return nil + }, } cmd.AddCommand( VersionCommand(), diff --git a/ecs/go.mod b/ecs/go.mod index 56489033..74a6a89f 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -50,6 +50,7 @@ require ( gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect + gotest.tools v2.2.0+incompatible gotest.tools/v3 v3.0.2 vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect ) diff --git a/ecs/pkg/docker/contextStore.go b/ecs/pkg/docker/contextStore.go index 2d40b9dd..31e4bc4e 100644 --- a/ecs/pkg/docker/contextStore.go +++ b/ecs/pkg/docker/contextStore.go @@ -3,6 +3,7 @@ package docker import ( "fmt" + "github.com/docker/cli/cli/command" cliconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/context/store" "github.com/mitchellh/mapstructure" @@ -25,7 +26,12 @@ type AwsContext struct { } func NewContext(name string, awsContext *AwsContext) error { - contextStore := initContextStore() + _, err := NewContextWithStore(name, awsContext, cliconfig.ContextStoreDir()) + return err +} + +func NewContextWithStore(name string, awsContext *AwsContext, contextDirectory string) (store.Store, error) { + contextStore := initContextStore(contextDirectory) endpoints := map[string]interface{}{ "aws": awsContext, "docker": awsContext, @@ -36,16 +42,19 @@ func NewContext(name string, awsContext *AwsContext) error { Endpoints: endpoints, Metadata: TypeContext{Type: contextType}, } - return contextStore.CreateOrUpdate(metadata) + return contextStore, contextStore.CreateOrUpdate(metadata) } -func initContextStore() store.Store { +func initContextStore(contextDir string) store.Store { config := store.NewConfig(getter) - return store.New(cliconfig.ContextStoreDir(), config) + return store.New(contextDir, config) } func CheckAwsContextExists(contextName string) (*AwsContext, error) { - contextStore := initContextStore() + if contextName == command.DefaultContextName { + return nil, fmt.Errorf("can't use \"%s\" with ECS command because it is not an AWS context", contextName) + } + contextStore := initContextStore(cliconfig.ContextStoreDir()) metadata, err := contextStore.GetMetadata(contextName) if err != nil { return nil, err diff --git a/ecs/tests/command_test.go b/ecs/tests/command_test.go new file mode 100644 index 00000000..a027fbae --- /dev/null +++ b/ecs/tests/command_test.go @@ -0,0 +1,18 @@ +package tests + +import ( + "testing" + + "gotest.tools/v3/icmd" +) + +func TestExitErrorCode(t *testing.T) { + cmd, cleanup := dockerCli.createTestCmd() + defer cleanup() + + cmd.Command = dockerCli.Command("ecs", "unknown_command") + icmd.RunCmd(cmd).Assert(t, icmd.Expected{ + ExitCode: 1, + Err: "\"unknown_command\" is not a docker ecs command\nSee 'docker ecs --help'", + }) +} diff --git a/ecs/tests/main_test.go b/ecs/tests/main_test.go new file mode 100644 index 00000000..a093c6c9 --- /dev/null +++ b/ecs/tests/main_test.go @@ -0,0 +1,127 @@ +package tests + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "testing" + + dockerConfigFile "github.com/docker/cli/cli/config/configfile" + "github.com/docker/ecs-plugin/pkg/docker" + "gotest.tools/v3/icmd" +) + +var ( + e2ePath = flag.String("e2e-path", ".", "Set path to the e2e directory") + dockerCliPath = os.Getenv("DOCKERCLI_BINARY") + dockerCli dockerCliCommand + testContextName = "testAwsContextToBeRemoved" +) + +type dockerCliCommand struct { + path string + cliPluginDir string +} + +type ConfigFileOperator func(configFile *dockerConfigFile.ConfigFile) + +func (d dockerCliCommand) createTestCmd(ops ...ConfigFileOperator) (icmd.Cmd, func()) { + configDir, err := ioutil.TempDir("", "config") + if err != nil { + panic(err) + } + configFilePath := filepath.Join(configDir, "config.json") + config := dockerConfigFile.ConfigFile{ + CLIPluginsExtraDirs: []string{ + d.cliPluginDir, + }, + Filename: configFilePath, + } + for _, op := range ops { + op(&config) + } + configFile, err := os.Create(configFilePath) + if err != nil { + panic(err) + } + defer configFile.Close() + err = json.NewEncoder(configFile).Encode(config) + if err != nil { + panic(err) + } + + awsContext := docker.AwsContext{ + Profile: "TestProfile", + Cluster: "TestCluster", + Region: "TestRegion", + } + testStore, err := docker.NewContextWithStore(testContextName, &awsContext, filepath.Join(configDir, "contexts")) + if err != nil { + panic(err) + } + cleanup := func() { + fmt.Println("cleanup") + testStore.Remove(testContextName) + os.RemoveAll(configDir) + } + env := append(os.Environ(), + "DOCKER_CONFIG="+configDir, + "DOCKER_CLI_EXPERIMENTAL=enabled") // TODO: Remove this once docker ecs plugin is no more experimental + return icmd.Cmd{Env: env}, cleanup +} + +func (d dockerCliCommand) Command(args ...string) []string { + return append([]string{d.path, "--context", testContextName}, args...) +} + +func TestMain(m *testing.M) { + flag.Parse() + if err := os.Chdir(*e2ePath); err != nil { + panic(err) + } + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + dockerEcs := os.Getenv("DOCKERECS_BINARY") + if dockerEcs == "" { + dockerEcs = filepath.Join(cwd, "../dist/docker-ecs") + } + dockerEcs, err = filepath.Abs(dockerEcs) + if err != nil { + panic(err) + } + if dockerCliPath == "" { + dockerCliPath = "docker" + } else { + dockerCliPath, err = filepath.Abs(dockerCliPath) + if err != nil { + panic(err) + } + } + // Prepare docker cli to call the docker-ecs plugin binary: + // - Create a symbolic link with the dockerEcs binary to the plugin directory + cliPluginDir, err := ioutil.TempDir("", "configContent") + if err != nil { + panic(err) + } + defer os.RemoveAll(cliPluginDir) + createDockerECSSymLink(dockerEcs, cliPluginDir) + + dockerCli = dockerCliCommand{path: dockerCliPath, cliPluginDir: cliPluginDir} + os.Exit(m.Run()) +} + +func createDockerECSSymLink(dockerEcs, configDir string) { + dockerEcsExecName := "docker-ecs" + if runtime.GOOS == "windows" { + dockerEcsExecName += ".exe" + } + if err := os.Symlink(dockerEcs, filepath.Join(configDir, dockerEcsExecName)); err != nil { + panic(err) + } +} diff --git a/ecs/tests/plugin_test.go b/ecs/tests/plugin_test.go new file mode 100644 index 00000000..f519c252 --- /dev/null +++ b/ecs/tests/plugin_test.go @@ -0,0 +1,33 @@ +package tests + +import ( + "regexp" + "testing" + + "gotest.tools/assert" + "gotest.tools/v3/golden" + "gotest.tools/v3/icmd" +) + +func TestInvokePluginFromCLI(t *testing.T) { + cmd, cleanup := dockerCli.createTestCmd() + defer cleanup() + // docker --help should list app as a top command + cmd.Command = dockerCli.Command("--help") + icmd.RunCmd(cmd).Assert(t, icmd.Expected{ + Out: "ecs* Docker ECS (Docker Inc.,", + }) + + // docker app --help prints docker-app help + cmd.Command = dockerCli.Command("ecs", "--help") + usage := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() + + goldenFile := "plugin-usage.golden" + golden.Assert(t, usage, goldenFile) + + // docker info should print app version and short description + cmd.Command = dockerCli.Command("info") + re := regexp.MustCompile(`ecs: Docker ECS \(Docker Inc\., .*\)`) + output := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() + assert.Assert(t, re.MatchString(output)) +} diff --git a/ecs/tests/setup_command_test.go b/ecs/tests/setup_command_test.go new file mode 100644 index 00000000..308d3e7e --- /dev/null +++ b/ecs/tests/setup_command_test.go @@ -0,0 +1,34 @@ +package tests + +import ( + "strings" + "testing" + + "gotest.tools/assert" + "gotest.tools/v3/golden" + "gotest.tools/v3/icmd" +) + +func TestSetupMandatoryArguments(t *testing.T) { + cmd, cleanup := dockerCli.createTestCmd() + defer cleanup() + + cmd.Command = dockerCli.Command("ecs", "setup") + icmd.RunCmd(cmd).Assert(t, icmd.Expected{ + ExitCode: 1, + Err: "required flag(s) \"cluster\", \"profile\", \"region\" not set", + }) +} +func TestDefaultAwsContextName(t *testing.T) { + cmd, cleanup := dockerCli.createTestCmd() + defer cleanup() + + cmd.Command = dockerCli.Command("ecs", "setup", "--cluster", "clusterName", "--profile", "profileName", + "--region", "regionName") + icmd.RunCmd(cmd).Assert(t, icmd.Success) + + cmd.Command = dockerCli.Command("context", "inspect", "aws") + output := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() + expected := golden.Get(t, "context-inspect.golden") + assert.Assert(t, strings.HasPrefix(output, string(expected))) +} diff --git a/ecs/tests/testdata/context-inspect.golden b/ecs/tests/testdata/context-inspect.golden new file mode 100644 index 00000000..ff61b55f --- /dev/null +++ b/ecs/tests/testdata/context-inspect.golden @@ -0,0 +1,16 @@ +[ + { + "Name": "aws", + "Metadata": {}, + "Endpoints": { + "aws": { + "Cluster": "clusterName", + "Profile": "profileName", + "Region": "regionName" + }, + "docker": { + "SkipTLSVerify": false + } + }, + "TLSMaterial": {}, + "Storage": \ No newline at end of file diff --git a/ecs/tests/testdata/plugin-usage.golden b/ecs/tests/testdata/plugin-usage.golden new file mode 100644 index 00000000..8114b6f4 --- /dev/null +++ b/ecs/tests/testdata/plugin-usage.golden @@ -0,0 +1,14 @@ + +Usage: docker ecs COMMAND + +run multi-container applications on Amazon ECS. + +Management Commands: + compose + secret Manages secrets + +Commands: + setup + version Show version. + +Run 'docker ecs COMMAND --help' for more information on a command. From 1889d04d831dd94ce929eb610409fbcebed26010 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 5 May 2020 14:45:34 +0200 Subject: [PATCH 055/205] Implement "network" using SecurityGroups Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/client.go | 1 + ecs/pkg/amazon/cloudformation.go | 74 +++++++++++++++++++++++--------- ecs/pkg/amazon/validate.go | 9 ++++ 3 files changed, 63 insertions(+), 21 deletions(-) diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index 2f6fd433..53b40af6 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -8,6 +8,7 @@ import ( const ( ProjectTag = "com.docker.compose.project" + NetworkTag = "com.docker.compose.network" ) func NewClient(profile string, cluster string, region string) (compose.API, error) { diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 4a9fcda9..45740b48 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -6,14 +6,16 @@ import ( "fmt" "strings" - ecsapi "github.com/aws/aws-sdk-go/service/ecs" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" + + ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/awslabs/goformation/v4/cloudformation/logs" cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" + "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -29,25 +31,9 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo return nil, err } - var ingresses = []ec2.SecurityGroup_Ingress{} - for _, service := range project.Services { - for _, port := range service.Ports { - ingresses = append(ingresses, ec2.SecurityGroup_Ingress{ - CidrIp: "0.0.0.0/0", - Description: fmt.Sprintf("%s:%d/%s", service.Name, port.Target, port.Protocol), - FromPort: int(port.Target), - IpProtocol: strings.ToUpper(port.Protocol), - ToPort: int(port.Target), - }) - } - } - - securityGroup := fmt.Sprintf("%s Security Group", project.Name) - template.Resources["SecurityGroup"] = &ec2.SecurityGroup{ - GroupDescription: securityGroup, - GroupName: securityGroup, - SecurityGroupIngress: ingresses, - VpcId: vpc, + for net := range project.Networks { + name, resource := convertNetwork(project, net, vpc) + template.Resources[name] = resource } logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) @@ -104,6 +90,12 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo }, } + serviceSecurityGroups := []string{} + for net := range service.Networks { + logicalName := networkResourceName(project, net) + serviceSecurityGroups = append(serviceSecurityGroups, cloudformation.Ref(logicalName)) + } + template.Resources[fmt.Sprintf("%sService", service.Name)] = &ecs.Service{ Cluster: c.Cluster, DesiredCount: 1, @@ -111,7 +103,7 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo NetworkConfiguration: &ecs.Service_NetworkConfiguration{ AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: ecsapi.AssignPublicIpEnabled, - SecurityGroups: []string{cloudformation.Ref("SecurityGroup")}, + SecurityGroups: serviceSecurityGroups, Subnets: subnets, }, }, @@ -128,6 +120,46 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo return template, nil } +func convertNetwork(project *compose.Project, net string, vpc string) (string, cloudformation.Resource) { + var ingresses []ec2.SecurityGroup_Ingress + for _, service := range project.Services { + if _, ok := service.Networks[net]; ok { + for _, port := range service.Ports { + ingresses = append(ingresses, ec2.SecurityGroup_Ingress{ + CidrIp: "0.0.0.0/0", + Description: fmt.Sprintf("%s:%d/%s", service.Name, port.Target, port.Protocol), + FromPort: int(port.Target), + IpProtocol: strings.ToUpper(port.Protocol), + ToPort: int(port.Target), + }) + } + } + } + + securityGroup := networkResourceName(project, net) + resource := &ec2.SecurityGroup{ + GroupDescription: fmt.Sprintf("%s %s Security Group", project.Name, net), + GroupName: securityGroup, + SecurityGroupIngress: ingresses, + VpcId: vpc, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + { + Key: NetworkTag, + Value: net, + }, + }, + } + return securityGroup, resource +} + +func networkResourceName(project *compose.Project, network string) string { + return fmt.Sprintf("%s%sNetwork", project.Name, strings.Title(network)) +} + func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, error) { //check compose file for the default external network if net, ok := project.Networks["default"]; ok { diff --git a/ecs/pkg/amazon/validate.go b/ecs/pkg/amazon/validate.go index 5e61a273..551dc7bc 100644 --- a/ecs/pkg/amazon/validate.go +++ b/ecs/pkg/amazon/validate.go @@ -15,6 +15,15 @@ func (c *client) Validate(project *compose.Project) error { } } + for i, service := range project.Services { + if len(service.Networks) == 0 { + // Service without explicit network attachment are implicitly exposed on default network + // FIXME move this to compose-go + service.Networks = map[string]*types.ServiceNetworkConfig{"default": nil} + project.Services[i] = service + } + } + // Here we can check for incompatible attributes, inject sane defaults, etc return nil } From 3e785e2cb055fa226e31d8c530e6b19232bbe942 Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Wed, 6 May 2020 12:28:33 +0200 Subject: [PATCH 056/205] Fix initialization issue of aws context with PreRun function Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 27 +++++++++++++++++--------- ecs/cmd/commands/secret.go | 35 +++++++++++++++++----------------- ecs/cmd/main/main.go | 17 +++-------------- ecs/pkg/docker/contextStore.go | 20 ++++++++++++++++++- 4 files changed, 58 insertions(+), 41 deletions(-) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 459a4881..06fc80a8 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -4,13 +4,14 @@ import ( "context" "fmt" + "github.com/docker/cli/cli/command" "github.com/docker/ecs-plugin/pkg/amazon" "github.com/docker/ecs-plugin/pkg/compose" "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" ) -func ComposeCommand(clusteropts *docker.AwsContext) *cobra.Command { +func ComposeCommand(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "compose", } @@ -18,9 +19,9 @@ func ComposeCommand(clusteropts *docker.AwsContext) *cobra.Command { opts.AddFlags(cmd.Flags()) cmd.AddCommand( - ConvertCommand(clusteropts, opts), - UpCommand(clusteropts, opts), - DownCommand(clusteropts, opts), + ConvertCommand(dockerCli, opts), + UpCommand(dockerCli, opts), + DownCommand(dockerCli, opts), ) return cmd } @@ -36,10 +37,14 @@ func (o upOptions) LoadBalancerArn() *string { return &o.loadBalancerArn } -func ConvertCommand(clusteropts *docker.AwsContext, projectOpts *compose.ProjectOptions) *cobra.Command { +func ConvertCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "convert", RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + clusteropts, err := docker.GetAwsContext(dockerCli) + if err != nil { + return err + } client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err @@ -61,11 +66,15 @@ func ConvertCommand(clusteropts *docker.AwsContext, projectOpts *compose.Project return cmd } -func UpCommand(clusteropts *docker.AwsContext, projectOpts *compose.ProjectOptions) *cobra.Command { +func UpCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ Use: "up", RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + clusteropts, err := docker.GetAwsContext(dockerCli) + if err != nil { + return err + } client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err @@ -81,11 +90,11 @@ type downOptions struct { DeleteCluster bool } -func DownCommand(clusteropts *docker.AwsContext, projectOpts *compose.ProjectOptions) *cobra.Command { +func DownCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { opts := downOptions{} cmd := &cobra.Command{ Use: "down", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err @@ -105,7 +114,7 @@ func DownCommand(clusteropts *docker.AwsContext, projectOpts *compose.ProjectOpt } } return nil - }, + }), } cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") return cmd diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go index b46eeeb3..8488e6cb 100644 --- a/ecs/cmd/commands/secret.go +++ b/ecs/cmd/commands/secret.go @@ -9,6 +9,7 @@ import ( "strings" "text/tabwriter" + "github.com/docker/cli/cli/command" "github.com/docker/ecs-plugin/pkg/amazon" "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" @@ -22,27 +23,27 @@ type deleteSecretOptions struct { recover bool } -func SecretCommand(clusteropts *docker.AwsContext) *cobra.Command { +func SecretCommand(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "secret", Short: "Manages secrets", } cmd.AddCommand( - CreateSecret(clusteropts), - InspectSecret(clusteropts), - ListSecrets(clusteropts), - DeleteSecret(clusteropts), + CreateSecret(dockerCli), + InspectSecret(dockerCli), + ListSecrets(dockerCli), + DeleteSecret(dockerCli), ) return cmd } -func CreateSecret(clusteropts *docker.AwsContext) *cobra.Command { +func CreateSecret(dockerCli command.Cli) *cobra.Command { //opts := createSecretOptions{} cmd := &cobra.Command{ Use: "create NAME SECRET", Short: "Creates a secret.", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err @@ -55,16 +56,16 @@ func CreateSecret(clusteropts *docker.AwsContext) *cobra.Command { id, err := client.CreateSecret(context.Background(), name, secret) fmt.Println(id) return err - }, + }), } return cmd } -func InspectSecret(clusteropts *docker.AwsContext) *cobra.Command { +func InspectSecret(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "inspect ID", Short: "Displays secret details", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err @@ -83,17 +84,17 @@ func InspectSecret(clusteropts *docker.AwsContext) *cobra.Command { } fmt.Println(out) return nil - }, + }), } return cmd } -func ListSecrets(clusteropts *docker.AwsContext) *cobra.Command { +func ListSecrets(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "list", Aliases: []string{"ls"}, Short: "List secrets stored for the existing account.", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err @@ -105,18 +106,18 @@ func ListSecrets(clusteropts *docker.AwsContext) *cobra.Command { printList(os.Stdout, secrets) return nil - }, + }), } return cmd } -func DeleteSecret(clusteropts *docker.AwsContext) *cobra.Command { +func DeleteSecret(dockerCli command.Cli) *cobra.Command { opts := deleteSecretOptions{} cmd := &cobra.Command{ Use: "delete NAME", Aliases: []string{"rm", "remove"}, Short: "Removes a secret.", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err @@ -125,7 +126,7 @@ func DeleteSecret(clusteropts *docker.AwsContext) *cobra.Command { return errors.New("Missing mandatory parameter: [NAME]") } return client.DeleteSecret(context.Background(), args[0], opts.recover) - }, + }), } cmd.Flags().BoolVar(&opts.recover, "recover", false, "Enable recovery.") return cmd diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 812ccfee..9af64829 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -7,7 +7,6 @@ import ( "github.com/docker/cli/cli-plugins/plugin" "github.com/docker/cli/cli/command" commands "github.com/docker/ecs-plugin/cmd/commands" - "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" ) @@ -27,33 +26,23 @@ func main() { // NewRootCmd returns the base root command. func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { - var opts *docker.AwsContext - cmd := &cobra.Command{ Short: "Docker ECS", Long: `run multi-container applications on Amazon ECS.`, Use: name, Annotations: map[string]string{"experimentalCLI": "true"}, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - err := plugin.PersistentPreRunE(cmd, args) - if err != nil { - return err - } - contextName := dockerCli.CurrentContext() - opts, err = docker.CheckAwsContextExists(contextName) - return err - }, RunE: func(cmd *cobra.Command, args []string) error { if len(args) != 0 { return fmt.Errorf("%q is not a docker ecs command\nSee 'docker ecs --help'", args[0]) } + cmd.Help() return nil }, } cmd.AddCommand( VersionCommand(), - commands.ComposeCommand(opts), - commands.SecretCommand(opts), + commands.ComposeCommand(dockerCli), + commands.SecretCommand(dockerCli), commands.SetupCommand(), ) return cmd diff --git a/ecs/pkg/docker/contextStore.go b/ecs/pkg/docker/contextStore.go index 31e4bc4e..b038eb92 100644 --- a/ecs/pkg/docker/contextStore.go +++ b/ecs/pkg/docker/contextStore.go @@ -7,6 +7,7 @@ import ( cliconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/context/store" "github.com/mitchellh/mapstructure" + "github.com/spf13/cobra" ) const contextType = "aws" @@ -50,7 +51,7 @@ func initContextStore(contextDir string) store.Store { return store.New(contextDir, config) } -func CheckAwsContextExists(contextName string) (*AwsContext, error) { +func checkAwsContextExists(contextName string) (*AwsContext, error) { if contextName == command.DefaultContextName { return nil, fmt.Errorf("can't use \"%s\" with ECS command because it is not an AWS context", contextName) } @@ -70,3 +71,20 @@ func CheckAwsContextExists(contextName string) (*AwsContext, error) { } return &awsContext, nil } + +type ContextFunc func(ctx AwsContext, args []string) error + +func WithAwsContext(dockerCli command.Cli, f ContextFunc) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + ctx, err := GetAwsContext(dockerCli) + if err != nil { + return err + } + return f(*ctx, args) + } +} + +func GetAwsContext(dockerCli command.Cli) (*AwsContext, error) { + contextName := dockerCli.CurrentContext() + return checkAwsContextExists(contextName) +} From f95bd4fdbf4859ba56ffa097eb3a581f640e6b9c Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 6 May 2020 18:36:22 +0200 Subject: [PATCH 057/205] mapping cpu and memory to fargate values Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/convert.go | 102 +++++++++++++++----------------------- 1 file changed, 40 insertions(+), 62 deletions(-) diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index a0bd9e42..6021b882 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -1,6 +1,8 @@ package amazon import ( + "errors" + "strconv" "strings" "time" @@ -13,7 +15,7 @@ import ( ) func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { - _, err := toCPULimits(service) + cpu, mem, err := toLimits(service) if err != nil { return nil, err } @@ -23,7 +25,6 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe // Here we can declare sidecars and init-containers using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_dependson { Command: service.Command, - Cpu: 256, DisableNetworking: service.NetworkMode == "none", DnsSearchDomains: service.DNSSearch, DnsServers: service.DNS, @@ -48,8 +49,6 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe "awslogs-stream-prefix": service.Name, }, }, - Memory: toMemoryLimits(service.Deploy), - MemoryReservation: toMemoryReservation(service.Deploy), MountPoints: nil, Name: service.Name, PortMappings: toPortMappings(service.Ports), @@ -68,10 +67,10 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe WorkingDirectory: service.WorkingDir, }, }, - Cpu: toCPU(service), + Cpu: cpu, Family: project.Name, IpcMode: service.Ipc, - Memory: toMemory(service), + Memory: mem, NetworkMode: ecsapi.NetworkModeAwsvpc, // FIXME could be set by service.NetworkMode, Fargate only supports network mode ‘awsvpc’. PidMode: service.Pid, PlacementConstraints: toPlacementConstraints(service.Deploy), @@ -82,32 +81,48 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe }, nil } -func toCPU(service types.ServiceConfig) string { - // FIXME based on service's memory/cpu requirements, select the adequate Fargate CPU - return "256" -} +func toLimits(service types.ServiceConfig) (string, string, error) { + // All possible cpu/mem values for Fargate + cpuToMem := map[int64][]types.UnitBytes{ + 256: {512, 1024, 2048}, + 512: {1024, 2048, 3072, 4096}, + 1024: {2048, 3072, 4096, 5120, 6144, 7168, 8192}, + 2048: {4096, 5120, 6144, 7168, 8192, 9216, 10240, 11264, 12288, 13312, 14336, 15360, 16384}, + 4096: {8192, 9216, 10240, 11264, 12288, 13312, 14336, 15360, 16384, 17408, 18432, 19456, 20480, 21504, 22528, 23552, 24576, 25600, 26624, 27648, 28672, 29696, 30720}, + } + cpuLimit := "256" + memLimit := "512" -func toMemory(service types.ServiceConfig) string { - // FIXME based on service's memory/cpu requirements, select the adequate Fargate CPU - return "512" -} - -func toCPULimits(service types.ServiceConfig) (*int64, error) { if service.Deploy == nil { - return nil, nil + return cpuLimit, memLimit, nil } - res := service.Deploy.Resources.Limits - if res == nil { - return nil, nil + + limits := service.Deploy.Resources.Limits + if limits == nil { + return cpuLimit, memLimit, nil } - if res.NanoCPUs == "" { - return nil, nil + + if limits.NanoCPUs == "" { + return cpuLimit, memLimit, nil } - v, err := opts.ParseCPUs(res.NanoCPUs) + + v, err := opts.ParseCPUs(limits.NanoCPUs) if err != nil { - return nil, err + return "", "", err } - return &v, nil + + for cpu, mem := range cpuToMem { + if v <= cpu*1024*1024 { + for _, m := range mem { + if limits.MemoryBytes <= m*1024*1024 { + cpuLimit = strconv.FormatInt(cpu, 10) + memLimit = strconv.FormatInt(int64(m), 10) + return cpuLimit, memLimit, nil + } + } + } + } + return "", "", errors.New("unable to find cpu/mem for the required resources") } func toRequiresCompatibilities(isolation string) []*string { @@ -117,19 +132,6 @@ func toRequiresCompatibilities(isolation string) []*string { return []*string{&isolation} } -func hasMemoryOrMemoryReservation(service types.ServiceConfig) bool { - if service.Deploy == nil { - return false - } - if service.Deploy.Resources.Reservations != nil { - return true - } - if service.Deploy.Resources.Limits != nil { - return true - } - return false -} - func toPlacementConstraints(deploy *types.DeployConfig) []ecs.TaskDefinition_TaskDefinitionPlacementConstraint { if deploy == nil || deploy.Placement.Constraints == nil || len(deploy.Placement.Constraints) == 0 { return nil @@ -176,30 +178,6 @@ func toUlimits(ulimits map[string]*types.UlimitsConfig) []ecs.TaskDefinition_Uli const Mb = 1024 * 1024 -func toMemoryLimits(deploy *types.DeployConfig) int { - if deploy == nil { - return 0 - } - res := deploy.Resources.Limits - if res == nil { - return 0 - } - v := int(res.MemoryBytes) / Mb - return v -} - -func toMemoryReservation(deploy *types.DeployConfig) int { - if deploy == nil { - return 0 - } - res := deploy.Resources.Reservations - if res == nil { - return 0 - } - v := int(res.MemoryBytes) / Mb - return v -} - func toLinuxParameters(service types.ServiceConfig) *ecs.TaskDefinition_LinuxParameters { return &ecs.TaskDefinition_LinuxParameters{ Capabilities: toKernelCapabilities(service.CapAdd, service.CapDrop), From 57d7474f7d3dc97b41755a4497d5fde4168f7923 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 30 Apr 2020 17:31:25 +0200 Subject: [PATCH 058/205] set secrets in cloudformation template Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/convert.go | 42 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 6021b882..4d75072c 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -19,7 +19,14 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe if err != nil { return nil, err } - + credential, err := getRepoCredentials(service) + if err != nil { + return nil, err + } + secrets, err := getSecrets(service) + if err != nil { + return nil, err + } return &ecs.TaskDefinition{ ContainerDefinitions: []ecs.TaskDefinition_ContainerDefinition{ // Here we can declare sidecars and init-containers using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_dependson @@ -55,9 +62,9 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe Privileged: service.Privileged, PseudoTerminal: service.Tty, ReadonlyRootFilesystem: service.ReadOnly, - RepositoryCredentials: nil, + RepositoryCredentials: credential, ResourceRequirements: nil, - Secrets: nil, + Secrets: secrets, StartTimeout: 0, StopTimeout: durationToInt(service.StopGracePeriod), SystemControls: nil, @@ -274,3 +281,32 @@ func toKeyValuePair(environment types.MappingWithEquals) []ecs.TaskDefinition_Ke } return pairs } + +func getRepoCredentials(service types.ServiceConfig) (*ecs.TaskDefinition_RepositoryCredentials, error) { + // extract registry and namespace string from image name + fields := strings.Split(service.Image, "/") + regPath := "" + for i, field := range fields { + if i < len(fields)-1 { + regPath = regPath + field + } + } + if regPath == "" || len(service.Secrets) == 0 { + return nil, nil + } + for _, secret := range service.Secrets { + if secret.Source == regPath { + return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: secret.Target}, nil + } + } + return nil, nil +} + +func getSecrets(service types.ServiceConfig) ([]ecs.TaskDefinition_Secret, error) { + secrets := []ecs.TaskDefinition_Secret{} + + for _, secret := range service.Secrets { + secrets = append(secrets, ecs.TaskDefinition_Secret{Name: secret.Target}) + } + return secrets, nil +} From d09c8c7236749a1bcb6af056d0aed15ed7d49f02 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 5 May 2020 18:55:03 +0200 Subject: [PATCH 059/205] add private images support Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/secret.go | 17 +++++++---- ecs/pkg/amazon/cloudformation.go | 48 ++++++++++++++++++++++++++++---- ecs/pkg/amazon/convert.go | 33 ++++++++++++---------- ecs/pkg/amazon/mock/api.go | 11 ++++---- ecs/pkg/amazon/sdk.go | 15 ++++++++-- ecs/pkg/amazon/secrets.go | 6 ++-- ecs/pkg/compose/api.go | 2 +- ecs/pkg/docker/secret.go | 23 +++++++++++++++ 8 files changed, 119 insertions(+), 36 deletions(-) diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go index 8488e6cb..f964f1ea 100644 --- a/ecs/cmd/commands/secret.go +++ b/ecs/cmd/commands/secret.go @@ -16,7 +16,10 @@ import ( ) type createSecretOptions struct { - Label string + Label string + Username string + Password string + Description string } type deleteSecretOptions struct { @@ -39,9 +42,9 @@ func SecretCommand(dockerCli command.Cli) *cobra.Command { } func CreateSecret(dockerCli command.Cli) *cobra.Command { - //opts := createSecretOptions{} + opts := createSecretOptions{} cmd := &cobra.Command{ - Use: "create NAME SECRET", + Use: "create NAME", Short: "Creates a secret.", RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) @@ -52,12 +55,16 @@ func CreateSecret(dockerCli command.Cli) *cobra.Command { return errors.New("Missing mandatory parameter: NAME") } name := args[0] - secret := args[1] - id, err := client.CreateSecret(context.Background(), name, secret) + + secret := docker.NewSecret(name, opts.Username, opts.Password, opts.Description) + id, err := client.CreateSecret(context.Background(), secret) fmt.Println(id) return err }), } + cmd.Flags().StringVarP(&opts.Username, "username", "u", "", "username") + cmd.Flags().StringVarP(&opts.Password, "password", "p", "", "password") + cmd.Flags().StringVarP(&opts.Description, "description", "d", "", "Secret description") return cmd } diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 45740b48..fa7f2f8a 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -55,17 +55,28 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo } taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", service.Name) + policy, err := c.getPolicy(ctx, definition) + if err != nil { + return nil, err + } + rolePolicies := []iam.Role_Policy{} + if policy != nil { + rolePolicies = append(rolePolicies, iam.Role_Policy{ + PolicyDocument: policy, + PolicyName: taskExecutionRole, + }) + + } + definition.ExecutionRoleArn = cloudformation.Ref(taskExecutionRole) + + taskDefinition := fmt.Sprintf("%sTaskDefinition", service.Name) template.Resources[taskExecutionRole] = &iam.Role{ AssumeRolePolicyDocument: assumeRolePolicyDocument, - // Here we can grant access to secrets/configs using a Policy { Allow,ssm:GetParameters,secret|config ARN} + Policies: rolePolicies, ManagedPolicyArns: []string{ ECSTaskExecutionPolicy, }, } - definition.ExecutionRoleArn = cloudformation.Ref(taskExecutionRole) - // FIXME definition.TaskRoleArn = ? - - taskDefinition := fmt.Sprintf("%sTaskDefinition", service.Name) template.Resources[taskDefinition] = definition var healthCheck *cloudmap.Service_HealthCheckConfig @@ -182,6 +193,33 @@ func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, e return defaultVPC, nil } +func (c client) getPolicy(ctx context.Context, taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { + + arns := []string{} + for _, container := range taskDef.ContainerDefinitions { + if container.RepositoryCredentials != nil { + arns = append(arns, container.RepositoryCredentials.CredentialsParameter) + } + if len(container.Secrets) > 0 { + for _, s := range container.Secrets { + arns = append(arns, s.ValueFrom) + } + } + + } + if len(arns) > 0 { + return &PolicyDocument{ + Statement: []PolicyStatement{ + { + Effect: "Allow", + Action: []string{"secretsmanager:GetSecretValue", "ssm:GetParameters", "kms:Decrypt"}, + Resource: arns, + }}, + }, nil + } + return nil, nil +} + type convertAPI interface { GetDefaultVPC(ctx context.Context) (string, error) VpcExists(ctx context.Context, vpcID string) (bool, error) diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 4d75072c..50c722ff 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -44,7 +44,7 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe FirelensConfiguration: nil, HealthCheck: toHealthCheck(service.HealthCheck), Hostname: service.Hostname, - Image: service.Image, + Image: getImage(service.Image), Interactive: false, Links: nil, LinuxParameters: toLinuxParameters(service), @@ -282,22 +282,27 @@ func toKeyValuePair(environment types.MappingWithEquals) []ecs.TaskDefinition_Ke return pairs } +func getImage(image string) string { + switch f := strings.Split(image, "/"); len(f) { + case 1: + return "docker.io/library/" + image + case 2: + return "docker.io/" + image + default: + return image + } +} + func getRepoCredentials(service types.ServiceConfig) (*ecs.TaskDefinition_RepositoryCredentials, error) { // extract registry and namespace string from image name - fields := strings.Split(service.Image, "/") - regPath := "" - for i, field := range fields { - if i < len(fields)-1 { - regPath = regPath + field + credential := "" + for key, value := range service.Extras { + if strings.HasPrefix(key, "x-aws-pull_credentials") { + credential = value.(string) } } - if regPath == "" || len(service.Secrets) == 0 { - return nil, nil - } - for _, secret := range service.Secrets { - if secret.Source == regPath { - return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: secret.Target}, nil - } + if credential != "" { + return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: credential}, nil } return nil, nil } @@ -306,7 +311,7 @@ func getSecrets(service types.ServiceConfig) ([]ecs.TaskDefinition_Secret, error secrets := []ecs.TaskDefinition_Secret{} for _, secret := range service.Secrets { - secrets = append(secrets, ecs.TaskDefinition_Secret{Name: secret.Target}) + secrets = append(secrets, ecs.TaskDefinition_Secret{Name: secret.Target, ValueFrom: secret.Source}) } return secrets, nil } diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index 4a116331..7eba9405 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -6,11 +6,12 @@ package mock import ( context "context" + reflect "reflect" + cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" docker "github.com/docker/ecs-plugin/pkg/docker" gomock "github.com/golang/mock/gomock" - reflect "reflect" ) // MockAPI is a mock of API interface @@ -67,18 +68,18 @@ func (mr *MockAPIMockRecorder) CreateCluster(arg0, arg1 interface{}) *gomock.Cal } // CreateSecret mocks base method -func (m *MockAPI) CreateSecret(arg0 context.Context, arg1, arg2 string) (string, error) { +func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 docker.Secret) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateSecret", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "CreateSecret", arg0, arg1) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateSecret indicates an expected call of CreateSecret -func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1 docker.Secret) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSecret", reflect.TypeOf((*MockAPI)(nil).CreateSecret), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSecret", reflect.TypeOf((*MockAPI)(nil).CreateSecret), arg0, arg1) } // CreateStack mocks base method diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 74cb0448..4bd9eea8 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -223,9 +223,18 @@ func (s sdk) DeleteStack(ctx context.Context, name string) error { return err } -func (s sdk) CreateSecret(ctx context.Context, name string, secret string) (string, error) { - logrus.Debug("Create secret " + name) - response, err := s.SM.CreateSecret(&secretsmanager.CreateSecretInput{Name: &name, SecretString: &secret}) +func (s sdk) CreateSecret(ctx context.Context, secret docker.Secret) (string, error) { + logrus.Debug("Create secret " + secret.Name) + secretStr, err := secret.GetCredString() + if err != nil { + return "", err + } + + response, err := s.SM.CreateSecret(&secretsmanager.CreateSecretInput{ + Name: &secret.Name, + SecretString: &secretStr, + Description: &secret.Description, + }) if err != nil { return "", err } diff --git a/ecs/pkg/amazon/secrets.go b/ecs/pkg/amazon/secrets.go index 649705f0..96a2a476 100644 --- a/ecs/pkg/amazon/secrets.go +++ b/ecs/pkg/amazon/secrets.go @@ -7,14 +7,14 @@ import ( ) type secretsAPI interface { - CreateSecret(ctx context.Context, name string, content string) (string, error) + CreateSecret(ctx context.Context, secret docker.Secret) (string, error) InspectSecret(ctx context.Context, id string) (docker.Secret, error) ListSecrets(ctx context.Context) ([]docker.Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error } -func (c client) CreateSecret(ctx context.Context, name string, content string) (string, error) { - return c.api.CreateSecret(ctx, name, content) +func (c client) CreateSecret(ctx context.Context, secret docker.Secret) (string, error) { + return c.api.CreateSecret(ctx, secret) } func (c client) InspectSecret(ctx context.Context, id string) (docker.Secret, error) { diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index e23651b6..6fd8409a 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -12,7 +12,7 @@ type API interface { ComposeUp(ctx context.Context, project *Project) error ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error - CreateSecret(ctx context.Context, name string, secret string) (string, error) + CreateSecret(ctx context.Context, secret docker.Secret) (string, error) InspectSecret(ctx context.Context, id string) (docker.Secret, error) ListSecrets(ctx context.Context) ([]docker.Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error diff --git a/ecs/pkg/docker/secret.go b/ecs/pkg/docker/secret.go index 0efae5d6..613c6263 100644 --- a/ecs/pkg/docker/secret.go +++ b/ecs/pkg/docker/secret.go @@ -9,6 +9,17 @@ type Secret struct { Name string `json:"Name"` Labels map[string]string `json:"Labels"` Description string `json:"Description"` + username string + password string +} + +func NewSecret(name, username, password, description string) Secret { + return Secret{ + Name: name, + username: username, + password: password, + Description: description, + } } func (s Secret) ToJSON() (string, error) { @@ -18,3 +29,15 @@ func (s Secret) ToJSON() (string, error) { } return string(b), nil } + +func (s Secret) GetCredString() (string, error) { + creds := map[string]string{ + "username": s.username, + "password": s.password, + } + b, err := json.Marshal(&creds) + if err != nil { + return "", err + } + return string(b), nil +} From 3a678fd7dc462f6516683a225409e61cef22f9e8 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 6 May 2020 15:15:46 +0200 Subject: [PATCH 060/205] cleanup Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 2 +- ecs/pkg/amazon/convert.go | 28 ++++++---------------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index fa7f2f8a..da041b85 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -63,7 +63,7 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo if policy != nil { rolePolicies = append(rolePolicies, iam.Role_Policy{ PolicyDocument: policy, - PolicyName: taskExecutionRole, + PolicyName: fmt.Sprintf("%sGrantAccessToSecrets", service.Name), }) } diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 50c722ff..1ccb0b02 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -19,14 +19,8 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe if err != nil { return nil, err } - credential, err := getRepoCredentials(service) - if err != nil { - return nil, err - } - secrets, err := getSecrets(service) - if err != nil { - return nil, err - } + credential := getRepoCredentials(service) + return &ecs.TaskDefinition{ ContainerDefinitions: []ecs.TaskDefinition_ContainerDefinition{ // Here we can declare sidecars and init-containers using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_dependson @@ -64,7 +58,6 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe ReadonlyRootFilesystem: service.ReadOnly, RepositoryCredentials: credential, ResourceRequirements: nil, - Secrets: secrets, StartTimeout: 0, StopTimeout: durationToInt(service.StopGracePeriod), SystemControls: nil, @@ -293,25 +286,16 @@ func getImage(image string) string { } } -func getRepoCredentials(service types.ServiceConfig) (*ecs.TaskDefinition_RepositoryCredentials, error) { +func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name credential := "" for key, value := range service.Extras { - if strings.HasPrefix(key, "x-aws-pull_credentials") { + if key == "x-aws-pull_credentials" { credential = value.(string) } } if credential != "" { - return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: credential}, nil + return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: credential} } - return nil, nil -} - -func getSecrets(service types.ServiceConfig) ([]ecs.TaskDefinition_Secret, error) { - secrets := []ecs.TaskDefinition_Secret{} - - for _, secret := range service.Secrets { - secrets = append(secrets, ecs.TaskDefinition_Secret{Name: secret.Target, ValueFrom: secret.Source}) - } - return secrets, nil + return nil } From 895dc249b455f0ffa3b7524a2d01c9bec2591aaa Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Fri, 8 May 2020 11:01:52 +0200 Subject: [PATCH 061/205] Manage aws credentials within setup command Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/setup.go | 45 +++++++++++++++++++++++++++++++++++++++ ecs/go.mod | 1 + ecs/go.sum | 2 ++ 3 files changed, 48 insertions(+) diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index a927605d..9beec37c 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -1,14 +1,23 @@ package commands import ( + "fmt" + "os" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/docker/cli/cli-plugins/plugin" contextStore "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" + "gopkg.in/ini.v1" ) func SetupCommand() *cobra.Command { var opts contextStore.AwsContext var name string + var accessKeyID string + var secretAccessKey string + cmd := &cobra.Command{ Use: "setup", Short: "", @@ -18,6 +27,11 @@ func SetupCommand() *cobra.Command { return plugin.PersistentPreRunE(cmd, args) }, RunE: func(cmd *cobra.Command, args []string) error { + if accessKeyID != "" && secretAccessKey != "" { + if err := saveCredentials(opts.Profile, accessKeyID, secretAccessKey); err != nil { + return err + } + } return contextStore.NewContext(name, &opts) }, } @@ -25,9 +39,40 @@ func SetupCommand() *cobra.Command { cmd.Flags().StringVarP(&opts.Profile, "profile", "p", "", "AWS Profile") cmd.Flags().StringVarP(&opts.Cluster, "cluster", "c", "", "ECS cluster") cmd.Flags().StringVarP(&opts.Region, "region", "r", "", "AWS region") + cmd.Flags().StringVarP(&accessKeyID, "aws-key-id", "k", "", "AWS Access Key ID") + cmd.Flags().StringVarP(&secretAccessKey, "aws-secret-key", "s", "", "AWS Secret Access Key") cmd.MarkFlagRequired("profile") cmd.MarkFlagRequired("cluster") cmd.MarkFlagRequired("region") return cmd } + +func saveCredentials(profile string, accessKeyID string, secretAccessKey string) error { + p := credentials.SharedCredentialsProvider{Profile: profile} + _, err := p.Retrieve() + if err == nil { + fmt.Println("credentials already exists!") + return nil + } + if err.(awserr.Error).Code() == "SharedCredsLoad" { + os.Create(p.Filename) + } + + credIni, err := ini.Load(p.Filename) + if err != nil { + return err + } + section := credIni.Section(profile) + section.Key("aws_access_key_id").SetValue(accessKeyID) + section.Key("aws_secret_access_key").SetValue(secretAccessKey) + + credFile, err := os.OpenFile(p.Filename, os.O_WRONLY, 0600) + if err != nil { + return err + } + if _, err = credIni.WriteTo(credFile); err != nil { + return err + } + return credFile.Close() +} diff --git a/ecs/go.mod b/ecs/go.mod index 74a6a89f..20f071d6 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -50,6 +50,7 @@ require ( gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect + gopkg.in/ini.v1 v1.55.0 gotest.tools v2.2.0+incompatible gotest.tools/v3 v3.0.2 vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect diff --git a/ecs/go.sum b/ecs/go.sum index 5cd52e76..5463d5c0 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -412,6 +412,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y2O4MU= gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= +gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ= +gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 51e04a4702651bcf4869623f3da2f612cd2fdf73 Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Sun, 10 May 2020 23:31:10 +0200 Subject: [PATCH 062/205] Add interactive context setup Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/setup.go | 241 ++++++++++++++++-- ecs/go.mod | 1 + ecs/go.sum | 15 ++ ecs/tests/setup_command_test.go | 7 +- .../testdata/setup-required-flags.golden | 13 + 5 files changed, 250 insertions(+), 27 deletions(-) create mode 100644 ecs/tests/testdata/setup-required-flags.golden diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index 9beec37c..7b93284f 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -3,20 +3,46 @@ package commands import ( "fmt" "os" + "reflect" + "strings" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/defaults" "github.com/docker/cli/cli-plugins/plugin" contextStore "github.com/docker/ecs-plugin/pkg/docker" + "github.com/manifoldco/promptui" "github.com/spf13/cobra" "gopkg.in/ini.v1" ) +const enterLabelPrefix = "Enter " + +type setupOptions struct { + name string + context contextStore.AwsContext + accessKeyID string + secretAccessKey string +} + +func (s setupOptions) unsetRequiredArgs() []string { + unset := []string{} + if s.context.Profile == "" { + unset = append(unset, "profile") + } + if s.context.Cluster == "" { + unset = append(unset, "cluster") + } + + if s.context.Region == "" { + unset = append(unset, "region") + } + return unset +} + func SetupCommand() *cobra.Command { - var opts contextStore.AwsContext - var name string - var accessKeyID string - var secretAccessKey string + var opts setupOptions + var interactive bool cmd := &cobra.Command{ Use: "setup", @@ -27,27 +53,63 @@ func SetupCommand() *cobra.Command { return plugin.PersistentPreRunE(cmd, args) }, RunE: func(cmd *cobra.Command, args []string) error { - if accessKeyID != "" && secretAccessKey != "" { - if err := saveCredentials(opts.Profile, accessKeyID, secretAccessKey); err != nil { + if interactive { + if err := interactiveCli(&opts); err != nil { + return err + } + } else { + if requiredFlag := opts.unsetRequiredArgs(); len(requiredFlag) > 0 { + fmt.Printf("required flag(s) %q not set", requiredFlag) + cmd.Help() + os.Exit(1) + } + } + if opts.accessKeyID != "" && opts.secretAccessKey != "" { + if err := saveCredentials(opts.context.Profile, opts.accessKeyID, opts.secretAccessKey); err != nil { return err } } - return contextStore.NewContext(name, &opts) + return contextStore.NewContext(opts.name, &opts.context) }, } - cmd.Flags().StringVarP(&name, "name", "n", "aws", "Context Name") - cmd.Flags().StringVarP(&opts.Profile, "profile", "p", "", "AWS Profile") - cmd.Flags().StringVarP(&opts.Cluster, "cluster", "c", "", "ECS cluster") - cmd.Flags().StringVarP(&opts.Region, "region", "r", "", "AWS region") - cmd.Flags().StringVarP(&accessKeyID, "aws-key-id", "k", "", "AWS Access Key ID") - cmd.Flags().StringVarP(&secretAccessKey, "aws-secret-key", "s", "", "AWS Secret Access Key") + cmd.Flags().StringVarP(&opts.name, "name", "n", "aws", "Context Name") + cmd.Flags().StringVarP(&opts.context.Profile, "profile", "p", "", "AWS Profile") + cmd.Flags().StringVarP(&opts.context.Cluster, "cluster", "c", "", "ECS cluster") + cmd.Flags().StringVarP(&opts.context.Region, "region", "r", "", "AWS region") + cmd.Flags().StringVarP(&opts.accessKeyID, "aws-key-id", "k", "", "AWS Access Key ID") + cmd.Flags().StringVarP(&opts.secretAccessKey, "aws-secret-key", "s", "", "AWS Secret Access Key") + cmd.Flags().BoolVarP(&interactive, "interactive", "", false, "Interactively setup Context and Credentials") - cmd.MarkFlagRequired("profile") - cmd.MarkFlagRequired("cluster") - cmd.MarkFlagRequired("region") return cmd } +func interactiveCli(opts *setupOptions) error { + var section ini.Section + + if err := setContextName(opts); err != nil { + return err + } + + section, err := setProfile(opts, section) + if err != nil { + return err + } + + if err := setCluster(opts, err); err != nil { + return err + } + + if err := setRegion(opts, section); err != nil { + return err + } + + if err := setCredentials(opts); err != nil { + return err + } + + return nil +} + func saveCredentials(profile string, accessKeyID string, secretAccessKey string) error { p := credentials.SharedCredentialsProvider{Profile: profile} _, err := p.Retrieve() @@ -55,7 +117,8 @@ func saveCredentials(profile string, accessKeyID string, secretAccessKey string) fmt.Println("credentials already exists!") return nil } - if err.(awserr.Error).Code() == "SharedCredsLoad" { + + if err.(awserr.Error).Code() == "SharedCredsLoad" && err.(awserr.Error).Message() == "failed to load shared credentials file" { os.Create(p.Filename) } @@ -63,16 +126,146 @@ func saveCredentials(profile string, accessKeyID string, secretAccessKey string) if err != nil { return err } - section := credIni.Section(profile) - section.Key("aws_access_key_id").SetValue(accessKeyID) - section.Key("aws_secret_access_key").SetValue(secretAccessKey) - - credFile, err := os.OpenFile(p.Filename, os.O_WRONLY, 0600) + section, err := credIni.NewSection(profile) if err != nil { return err } - if _, err = credIni.WriteTo(credFile); err != nil { + section.NewKey("aws_access_key_id", accessKeyID) + section.NewKey("aws_secret_access_key", secretAccessKey) + return credIni.SaveTo(p.Filename) +} + +func awsProfiles(filename string) (map[string]ini.Section, error) { + profiles := map[string]ini.Section{"new profile": {}} + if filename == "" { + filename = defaults.SharedConfigFilename() + } + credIni, err := ini.Load(filename) + if err != nil { + return nil, err + } + if err != nil { + return nil, err + } + for _, section := range credIni.Sections() { + if strings.HasPrefix(section.Name(), "profile") { + profiles[section.Name()[len("profile "):]] = *section + } + } + return profiles, nil +} + +func setContextName(opts *setupOptions) error { + if opts.name == "aws" { + result, err := promptString(opts.name, "context name", enterLabelPrefix, 2) + if err != nil { + return err + } + opts.name = result + } + return nil +} + +func setProfile(opts *setupOptions, section ini.Section) (ini.Section, error) { + profilesList, err := awsProfiles("") + if err != nil { + return ini.Section{}, err + } + section, ok := profilesList[opts.context.Profile] + if !ok { + prompt := promptui.Select{ + Label: "Select AWS Profile", + Items: reflect.ValueOf(profilesList).MapKeys(), + } + _, result, err := prompt.Run() + if result == "new profile" { + result, err := promptString(opts.context.Profile, "profile name", enterLabelPrefix, 2) + if err != nil { + return ini.Section{}, err + } + opts.context.Profile = result + } else { + section = profilesList[result] + opts.context.Profile = result + } + if err != nil { + return ini.Section{}, err + } + } + return section, nil +} + +func setRegion(opts *setupOptions, section ini.Section) error { + defaultRegion := opts.context.Region + if defaultRegion == "" && section.Name() != "" { + region, err := section.GetKey("region") + if err == nil { + defaultRegion = region.Value() + } + } + result, err := promptString(defaultRegion, "region", enterLabelPrefix, 2) + if err != nil { return err } - return credFile.Close() + opts.context.Region = result + return nil +} + +func setCluster(opts *setupOptions, err error) error { + result, err := promptString(opts.context.Cluster, "cluster name", enterLabelPrefix, 2) + if err != nil { + return err + } + opts.context.Cluster = result + return nil +} + +func setCredentials(opts *setupOptions) error { + prompt := promptui.Prompt{ + Label: "Enter credentials", + IsConfirm: true, + } + _, err := prompt.Run() + if err == nil { + result, err := promptString(opts.accessKeyID, "AWS Access Key ID", enterLabelPrefix, 3) + if err != nil { + return err + } + opts.accessKeyID = result + + prompt = promptui.Prompt{ + Label: "Enter AWS Secret Access Key", + Validate: validateMinLen("AWS Secret Access Key", 3), + Mask: '*', + Default: opts.secretAccessKey, + } + result, err = prompt.Run() + if err != nil { + return err + } + opts.secretAccessKey = result + } + return nil +} + +func promptString(defaultValue string, label string, labelPrefix string, minLength int) (string, error) { + prompt := promptui.Prompt{ + Label: labelPrefix + label, + Validate: validateMinLen(label, minLength), + Default: defaultValue, + } + result, err := prompt.Run() + if err != nil { + return "", err + } + return result, nil +} + +func validateMinLen(label string, minLength int) func(input string) error { + return func(input string) error { + if len(input) < minLength { + return fmt.Errorf("%s must have more than %d characters", label, minLength) + } + return nil + } } diff --git a/ecs/go.mod b/ecs/go.mod index 20f071d6..7d04a3de 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -33,6 +33,7 @@ require ( github.com/jinzhu/gorm v1.9.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lib/pq v1.3.0 // indirect + github.com/manifoldco/promptui v0.7.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/miekg/pkcs11 v1.0.3 // indirect github.com/mitchellh/mapstructure v1.2.2 diff --git a/ecs/go.sum b/ecs/go.sum index 5463d5c0..2151851e 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -42,6 +42,10 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiKw= @@ -161,6 +165,8 @@ github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVz github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= +github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= @@ -182,8 +188,16 @@ github.com/lib/pq v0.0.0-20180201184707-88edab080323/go.mod h1:5WUZQaWbwv1U+lTRe github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= +github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/manifoldco/promptui v0.7.0 h1:3l11YT8tm9MnwGFQ4kETwkzpAwY2Jt9lCrumCUW4+z4= +github.com/manifoldco/promptui v0.7.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -355,6 +369,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/ecs/tests/setup_command_test.go b/ecs/tests/setup_command_test.go index 308d3e7e..104ba229 100644 --- a/ecs/tests/setup_command_test.go +++ b/ecs/tests/setup_command_test.go @@ -14,10 +14,11 @@ func TestSetupMandatoryArguments(t *testing.T) { defer cleanup() cmd.Command = dockerCli.Command("ecs", "setup") - icmd.RunCmd(cmd).Assert(t, icmd.Expected{ + usage := icmd.RunCmd(cmd).Assert(t, icmd.Expected{ ExitCode: 1, - Err: "required flag(s) \"cluster\", \"profile\", \"region\" not set", - }) + }).Combined() + goldenFile := "setup-required-flags.golden" + golden.Assert(t, usage, goldenFile) } func TestDefaultAwsContextName(t *testing.T) { cmd, cleanup := dockerCli.createTestCmd() diff --git a/ecs/tests/testdata/setup-required-flags.golden b/ecs/tests/testdata/setup-required-flags.golden new file mode 100644 index 00000000..8666ef14 --- /dev/null +++ b/ecs/tests/testdata/setup-required-flags.golden @@ -0,0 +1,13 @@ +required flag(s) ["profile" "cluster" "region"] not set +Usage: docker ecs setup + + + +Options: + -k, --aws-key-id string AWS Access Key ID + -s, --aws-secret-key string AWS Secret Access Key + -c, --cluster string ECS cluster + --interactive Interactively setup Context and Credentials + -n, --name string Context Name (default "aws") + -p, --profile string AWS Profile + -r, --region string AWS region From 0864513bfeb9aa8a2bf2ee8e2172ca939c837b3e Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Mon, 11 May 2020 16:20:50 +0200 Subject: [PATCH 063/205] Switch automatically to interactive mode if one of the required flag is not set Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/setup.go | 10 +--------- ecs/tests/setup_command_test.go | 11 ----------- ecs/tests/testdata/setup-required-flags.golden | 13 ------------- 3 files changed, 1 insertion(+), 33 deletions(-) delete mode 100644 ecs/tests/testdata/setup-required-flags.golden diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index 7b93284f..a4d55e61 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -42,7 +42,6 @@ func (s setupOptions) unsetRequiredArgs() []string { func SetupCommand() *cobra.Command { var opts setupOptions - var interactive bool cmd := &cobra.Command{ Use: "setup", @@ -53,16 +52,10 @@ func SetupCommand() *cobra.Command { return plugin.PersistentPreRunE(cmd, args) }, RunE: func(cmd *cobra.Command, args []string) error { - if interactive { + if requiredFlag := opts.unsetRequiredArgs(); len(requiredFlag) > 0 { if err := interactiveCli(&opts); err != nil { return err } - } else { - if requiredFlag := opts.unsetRequiredArgs(); len(requiredFlag) > 0 { - fmt.Printf("required flag(s) %q not set", requiredFlag) - cmd.Help() - os.Exit(1) - } } if opts.accessKeyID != "" && opts.secretAccessKey != "" { if err := saveCredentials(opts.context.Profile, opts.accessKeyID, opts.secretAccessKey); err != nil { @@ -78,7 +71,6 @@ func SetupCommand() *cobra.Command { cmd.Flags().StringVarP(&opts.context.Region, "region", "r", "", "AWS region") cmd.Flags().StringVarP(&opts.accessKeyID, "aws-key-id", "k", "", "AWS Access Key ID") cmd.Flags().StringVarP(&opts.secretAccessKey, "aws-secret-key", "s", "", "AWS Secret Access Key") - cmd.Flags().BoolVarP(&interactive, "interactive", "", false, "Interactively setup Context and Credentials") return cmd } diff --git a/ecs/tests/setup_command_test.go b/ecs/tests/setup_command_test.go index 104ba229..a0cc9e46 100644 --- a/ecs/tests/setup_command_test.go +++ b/ecs/tests/setup_command_test.go @@ -9,17 +9,6 @@ import ( "gotest.tools/v3/icmd" ) -func TestSetupMandatoryArguments(t *testing.T) { - cmd, cleanup := dockerCli.createTestCmd() - defer cleanup() - - cmd.Command = dockerCli.Command("ecs", "setup") - usage := icmd.RunCmd(cmd).Assert(t, icmd.Expected{ - ExitCode: 1, - }).Combined() - goldenFile := "setup-required-flags.golden" - golden.Assert(t, usage, goldenFile) -} func TestDefaultAwsContextName(t *testing.T) { cmd, cleanup := dockerCli.createTestCmd() defer cleanup() diff --git a/ecs/tests/testdata/setup-required-flags.golden b/ecs/tests/testdata/setup-required-flags.golden deleted file mode 100644 index 8666ef14..00000000 --- a/ecs/tests/testdata/setup-required-flags.golden +++ /dev/null @@ -1,13 +0,0 @@ -required flag(s) ["profile" "cluster" "region"] not set -Usage: docker ecs setup - - - -Options: - -k, --aws-key-id string AWS Access Key ID - -s, --aws-secret-key string AWS Secret Access Key - -c, --cluster string ECS cluster - --interactive Interactively setup Context and Credentials - -n, --name string Context Name (default "aws") - -p, --profile string AWS Profile - -r, --region string AWS region From 4ab37f8229988a349b5090980827345b0169f7ff Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 12 May 2020 09:41:29 +0200 Subject: [PATCH 064/205] Implement Hostname-only service discovery using LOCALDOMAIN Signed-off-by: Nicolas De Loof --- ecs/README.md | 24 ++++++++++++++++++++++++ ecs/go.mod | 3 +-- ecs/go.sum | 7 +++++++ ecs/pkg/amazon/convert.go | 22 ++++++++++++++-------- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/ecs/README.md b/ecs/README.md index f15fd25f..afa354d3 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -39,3 +39,27 @@ file do not include unsupported features. * _Convert_ produces a CloudFormation template to define all resources required to implement the application model on AWS. * _Apply_ phase do apply the CloudFormation template, either by exporting to a stack file or to deploy on AWS. +## Application model + +### Services + +Compose services are mapped to ECS services. Compose specification has no support for multi-container services, nor +does it support sidecars. When an ECS feature requires a sidecar, we introduce custom Compose extension (`x-aws-*`) +to actually expose ECS feature as a service-level feature, not plumbing details. + +### Networking + +We map the "network" abstraction from Compose model to AWS SecurityGroups. The whole application is created within a +single VPC, SecurityGroups are created per networks, including the implicit `default` one. Services are attached +according to the networks declared in Compose model. Doing so, services attached to a common security group can +communicate together, while services from distinct SecurityGroups can't. We just can't set service aliasses per network. + +A CloudMap private namespace is created for application as `{project}.local`. Services get registered so that we +get service discovery and DNS round-robin (equivalent for Compose's `endpoint_mode: dnsrr`). Hostname-only service +discovery is enabled by running application containers with `LOCALDOMAIN={project}.local` +(see [resolv.conf(5)](http://man7.org/linux/man-pages/man5/resolv.conf.5.html)). This works out-of-the-box for +debian-based Docker images. Alpine images have to include a tiny entrypoint script to replicate this feature: +```shell script +if [ $LOCALDOMAIN ]; then echo "search ${LOCALDOMAIN}" >> /etc/resolv.conf; fi +``` + diff --git a/ecs/go.mod b/ecs/go.mod index 7d04a3de..7eedd06b 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -5,7 +5,7 @@ require ( github.com/Microsoft/hcsshim v0.8.7 // indirect github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect - github.com/aws/aws-sdk-go v1.28.9 + github.com/aws/aws-sdk-go v1.30.22 github.com/awslabs/goformation/v4 v4.8.0 github.com/bitly/go-hostpool v0.1.0 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect @@ -24,7 +24,6 @@ require ( github.com/docker/go v1.5.1-1 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect - github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.1 // indirect github.com/golang/mock v1.4.3 diff --git a/ecs/go.sum b/ecs/go.sum index 2151851e..1e4fd8fd 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -21,6 +21,9 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0= github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.30.22 h1:wImJ8jQrplgmxaTeUY7FrJFn4te/VtWq+mmmJ1TnWAg= +github.com/aws/aws-sdk-go v1.30.22/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/awslabs/goformation/v4 v4.8.0 h1:UiUhyokRy3suEqBXTnipvY8klqY3Eyl4GCH17brraEc= github.com/awslabs/goformation/v4 v4.8.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -161,6 +164,8 @@ github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -300,6 +305,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/theupdateframework/notary v0.6.1 h1:7wshjstgS9x9F5LuB1L5mBI2xNMObWqjz+cjWoom6l0= github.com/theupdateframework/notary v0.6.1/go.mod h1:MOfgIfmox8s7/7fduvB2xyPPMJCrjRLRizA8OFwpnKY= @@ -357,6 +363,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYR golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 1ccb0b02..13e90690 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -1,11 +1,12 @@ package amazon import ( - "errors" + "fmt" "strconv" "strings" "time" + "github.com/aws/aws-sdk-go/aws" ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ecs" @@ -21,9 +22,17 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe } credential := getRepoCredentials(service) + // override resolve.conf search directive to also search .local + // TODO remove once ECS support hostname-only service discovery + service.Environment["LOCALDOMAIN"] = aws.String( + cloudformation.Join("", []string{ + cloudformation.Ref("AWS::Region"), + ".compute.internal", + fmt.Sprintf(" %s.local", project.Name), + })) + return &ecs.TaskDefinition{ ContainerDefinitions: []ecs.TaskDefinition_ContainerDefinition{ - // Here we can declare sidecars and init-containers using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_dependson { Command: service.Command, DisableNetworking: service.NetworkMode == "none", @@ -50,7 +59,6 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe "awslogs-stream-prefix": service.Name, }, }, - MountPoints: nil, Name: service.Name, PortMappings: toPortMappings(service.Ports), Privileged: service.Privileged, @@ -68,7 +76,7 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe }, }, Cpu: cpu, - Family: project.Name, + Family: fmt.Sprintf("%s-%s", project.Name, service.Name), IpcMode: service.Ipc, Memory: mem, NetworkMode: ecsapi.NetworkModeAwsvpc, // FIXME could be set by service.NetworkMode, Fargate only supports network mode ‘awsvpc’. @@ -77,7 +85,6 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe ProxyConfiguration: nil, RequiresCompatibilities: []string{ecsapi.LaunchTypeFargate}, Tags: nil, - Volumes: []ecs.TaskDefinition_Volume{}, }, nil } @@ -122,7 +129,7 @@ func toLimits(service types.ServiceConfig) (string, string, error) { } } } - return "", "", errors.New("unable to find cpu/mem for the required resources") + return "", "", fmt.Errorf("unable to find cpu/mem for the required resources") } func toRequiresCompatibilities(isolation string) []*string { @@ -198,8 +205,7 @@ func toTmpfs(tmpfs types.StringList) []ecs.TaskDefinition_Tmpfs { for _, path := range tmpfs { o = append(o, ecs.TaskDefinition_Tmpfs{ ContainerPath: path, - MountOptions: nil, - Size: 0, + Size: 100, // size is required on ECS, unlimited by the compose spec }) } return o From 69a7ef076367cd97007a0245f3bbafcec09fe6e8 Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Mon, 11 May 2020 17:19:51 +0200 Subject: [PATCH 065/205] Make cluster optional in context setup Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/setup.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index a4d55e61..dd6c40f6 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -30,10 +30,6 @@ func (s setupOptions) unsetRequiredArgs() []string { if s.context.Profile == "" { unset = append(unset, "profile") } - if s.context.Cluster == "" { - unset = append(unset, "cluster") - } - if s.context.Region == "" { unset = append(unset, "region") } @@ -204,7 +200,7 @@ func setRegion(opts *setupOptions, section ini.Section) error { } func setCluster(opts *setupOptions, err error) error { - result, err := promptString(opts.context.Cluster, "cluster name", enterLabelPrefix, 2) + result, err := promptString(opts.context.Cluster, "cluster name", enterLabelPrefix, 0) if err != nil { return err } From 1fdac494f30258e0795d1a10531e4f633712b96f Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 12 May 2020 15:22:17 +0200 Subject: [PATCH 066/205] Create CloudFormation template with parameters so we don't need AWS API to resolve IDs and can run conversion offline Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 2 +- ecs/go.sum | 1 + ecs/pkg/amazon/api.go | 1 - ecs/pkg/amazon/cloudformation.go | 71 ++++++++++++-------------------- ecs/pkg/amazon/iam.go | 8 +++- ecs/pkg/amazon/mock/api.go | 28 +++---------- ecs/pkg/amazon/sdk.go | 12 +++++- ecs/pkg/amazon/up.go | 48 +++++++++++++++++++-- ecs/pkg/compose/api.go | 2 +- 9 files changed, 99 insertions(+), 74 deletions(-) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 06fc80a8..18a5bf81 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -49,7 +49,7 @@ func ConvertCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) if err != nil { return err } - template, err := client.Convert(context.Background(), project) + template, err := client.Convert(project) if err != nil { return err } diff --git a/ecs/go.sum b/ecs/go.sum index 1e4fd8fd..5ca9a4bf 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -305,6 +305,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/theupdateframework/notary v0.6.1 h1:7wshjstgS9x9F5LuB1L5mBI2xNMObWqjz+cjWoom6l0= diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index b4914d68..49306185 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -5,6 +5,5 @@ package amazon type API interface { downAPI upAPI - convertAPI secretsAPI } diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index da041b85..075037c0 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -1,8 +1,6 @@ package amazon import ( - "context" - "errors" "fmt" "strings" @@ -19,20 +17,31 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) -func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudformation.Template, error) { +func (c client) Convert(project *compose.Project) (*cloudformation.Template, error) { template := cloudformation.NewTemplate() - vpc, err := c.GetVPC(ctx, project) - if err != nil { - return nil, err + template.Parameters["VPCId"] = cloudformation.Parameter{ + Type: "AWS::EC2::VPC::Id", + Description: "ID of the VPC", } - subnets, err := c.api.GetSubNets(ctx, vpc) - if err != nil { - return nil, err + /* + FIXME can't set subnets: Ref("SubnetIds") see https://github.com/awslabs/goformation/issues/282 + template.Parameters["SubnetIds"] = cloudformation.Parameter{ + Type: "List", + Description: "The list of SubnetIds, for at least two Availability Zones in the region in your VPC", + } + */ + template.Parameters["Subnet1Id"] = cloudformation.Parameter{ + Type: "AWS::EC2::Subnet::Id", + Description: "SubnetId,for Availability Zone 1 in the region in your VPC", + } + template.Parameters["Subnet2Id"] = cloudformation.Parameter{ + Type: "AWS::EC2::Subnet::Id", + Description: "SubnetId,for Availability Zone 1 in the region in your VPC", } for net := range project.Networks { - name, resource := convertNetwork(project, net, vpc) + name, resource := convertNetwork(project, net, cloudformation.Ref("VPCId")) template.Resources[name] = resource } @@ -45,7 +54,7 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), Name: fmt.Sprintf("%s.local", project.Name), - Vpc: vpc, + Vpc: cloudformation.Ref("VPCId"), } for _, service := range project.Services { @@ -55,7 +64,7 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo } taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", service.Name) - policy, err := c.getPolicy(ctx, definition) + policy, err := c.getPolicy(definition) if err != nil { return nil, err } @@ -115,7 +124,10 @@ func (c client) Convert(ctx context.Context, project *compose.Project) (*cloudfo AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: ecsapi.AssignPublicIpEnabled, SecurityGroups: serviceSecurityGroups, - Subnets: subnets, + Subnets: []string{ + cloudformation.Ref("Subnet1Id"), + cloudformation.Ref("Subnet2Id"), + }, }, }, SchedulingStrategy: ecsapi.SchedulingStrategyReplica, @@ -171,29 +183,7 @@ func networkResourceName(project *compose.Project, network string) string { return fmt.Sprintf("%s%sNetwork", project.Name, strings.Title(network)) } -func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, error) { - //check compose file for the default external network - if net, ok := project.Networks["default"]; ok { - if net.External.External { - vpc := net.Name - ok, err := c.api.VpcExists(ctx, vpc) - if err != nil { - return "", err - } - if !ok { - return "", errors.New("Vpc does not exist: " + vpc) - } - return vpc, nil - } - } - defaultVPC, err := c.api.GetDefaultVPC(ctx) - if err != nil { - return "", err - } - return defaultVPC, nil -} - -func (c client) getPolicy(ctx context.Context, taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { +func (c client) getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { arns := []string{} for _, container := range taskDef.ContainerDefinitions { @@ -212,17 +202,10 @@ func (c client) getPolicy(ctx context.Context, taskDef *ecs.TaskDefinition) (*Po Statement: []PolicyStatement{ { Effect: "Allow", - Action: []string{"secretsmanager:GetSecretValue", "ssm:GetParameters", "kms:Decrypt"}, + Action: []string{ActionGetSecretValue, ActionGetParameters, ActionDecrypt}, Resource: arns, }}, }, nil } return nil, nil } - -type convertAPI interface { - GetDefaultVPC(ctx context.Context) (string, error) - VpcExists(ctx context.Context, vpcID string) (bool, error) - GetSubNets(ctx context.Context, vpcID string) ([]string, error) - GetRoleArn(ctx context.Context, name string) (string, error) -} diff --git a/ecs/pkg/amazon/iam.go b/ecs/pkg/amazon/iam.go index c07e34fe..66357730 100644 --- a/ecs/pkg/amazon/iam.go +++ b/ecs/pkg/amazon/iam.go @@ -1,6 +1,12 @@ package amazon -const ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +const ( + ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + + ActionGetSecretValue = "secretsmanager:GetSecretValue" + ActionGetParameters = "ssm:GetParameters" + ActionDecrypt = "kms:Decrypt" +) var assumeRolePolicyDocument = PolicyDocument{ Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index 7eba9405..1210a8d3 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -6,12 +6,11 @@ package mock import ( context "context" - reflect "reflect" - cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" docker "github.com/docker/ecs-plugin/pkg/docker" gomock "github.com/golang/mock/gomock" + reflect "reflect" ) // MockAPI is a mock of API interface @@ -77,23 +76,23 @@ func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 docker.Secret) (string } // CreateSecret indicates an expected call of CreateSecret -func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1 docker.Secret) *gomock.Call { +func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSecret", reflect.TypeOf((*MockAPI)(nil).CreateSecret), arg0, arg1) } // CreateStack mocks base method -func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation0.Template) error { +func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation0.Template, arg3 map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // CreateStack indicates an expected call of CreateStack -func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2, arg3) } // DeleteCluster mocks base method @@ -168,21 +167,6 @@ func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC), arg0) } -// GetRoleArn mocks base method -func (m *MockAPI) GetRoleArn(arg0 context.Context, arg1 string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRoleArn", arg0, arg1) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetRoleArn indicates an expected call of GetRoleArn -func (mr *MockAPIMockRecorder) GetRoleArn(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRoleArn", reflect.TypeOf((*MockAPI)(nil).GetRoleArn), arg0, arg1) -} - // GetStackID mocks base method func (m *MockAPI) GetStackID(arg0 context.Context, arg1 string) (string, error) { m.ctrl.T.Helper() diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 4bd9eea8..f0fc076f 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -153,17 +153,27 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) { return len(stacks.Stacks) > 0, nil } -func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template) error { +func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template, parameters map[string]string) error { logrus.Debug("Create CloudFormation stack") json, err := template.JSON() if err != nil { return err } + param := []*cloudformation.Parameter{} + for name, value := range parameters { + param = append(param, &cloudformation.Parameter{ + ParameterKey: aws.String(name), + ParameterValue: aws.String(value), + UsePreviousValue: aws.Bool(true), + }) + } + _, err = s.CF.CreateStackWithContext(ctx, &cloudformation.CreateStackInput{ OnFailure: aws.String("DELETE"), StackName: aws.String(name), TemplateBody: aws.String(string(json)), + Parameters: param, TimeoutInMinutes: aws.Int64(10), Capabilities: []*string{ aws.String(cloudformation.CapabilityCapabilityIam), diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 21adf973..e1a0f1fd 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -29,12 +29,28 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return err } - template, err := c.Convert(ctx, project) + template, err := c.Convert(project) if err != nil { return err } - err = c.api.CreateStack(ctx, project.Name, template) + vpc, err := c.GetVPC(ctx, project) + if err != nil { + return err + } + + subNets, err := c.api.GetSubNets(ctx, vpc) + if err != nil { + return err + } + + parameters := map[string]string{ + "VPCId": vpc, + "Subnet1Id": subNets[0], + "Subnet2Id": subNets[1], + } + + err = c.api.CreateStack(ctx, project.Name, template, parameters) if err != nil { return err } @@ -42,10 +58,36 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return c.WaitStackCompletion(ctx, project.Name, StackCreate) } +func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, error) { + //check compose file for the default external network + if net, ok := project.Networks["default"]; ok { + if net.External.External { + vpc := net.Name + ok, err := c.api.VpcExists(ctx, vpc) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("VPC does not exist: %s", vpc) + } + return vpc, nil + } + } + defaultVPC, err := c.api.GetDefaultVPC(ctx) + if err != nil { + return "", err + } + return defaultVPC, nil +} + type upAPI interface { waitAPI + GetDefaultVPC(ctx context.Context) (string, error) + VpcExists(ctx context.Context, vpcID string) (bool, error) + GetSubNets(ctx context.Context, vpcID string) ([]string, error) + ClusterExists(ctx context.Context, name string) (bool, error) CreateCluster(ctx context.Context, name string) (string, error) StackExists(ctx context.Context, name string) (bool, error) - CreateStack(ctx context.Context, name string, template *cloudformation.Template) error + CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error } diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 6fd8409a..70de9aa5 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -8,7 +8,7 @@ import ( ) type API interface { - Convert(ctx context.Context, project *Project) (*cloudformation.Template, error) + Convert(project *Project) (*cloudformation.Template, error) ComposeUp(ctx context.Context, project *Project) error ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error From 43d3d94c43e21166b4577a8c8b84d0a18ddff2a3 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 13 May 2020 10:34:23 +0200 Subject: [PATCH 067/205] Create cluster by compose up close #53 Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 45 ++++++++++++++++++++++++++------ ecs/pkg/amazon/up.go | 23 +++++++++------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 075037c0..d14b5faf 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -17,9 +17,23 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) +const ( + ParameterClusterName = "ParameterClusterName" + ParameterVPCId = "ParameterVPCId" + ParameterSubnet1Id = "ParameterSubnet1Id" + ParameterSubnet2Id = "ParameterSubnet2Id" +) + +// Convert a compose project into a CloudFormation template func (c client) Convert(project *compose.Project) (*cloudformation.Template, error) { template := cloudformation.NewTemplate() - template.Parameters["VPCId"] = cloudformation.Parameter{ + + template.Parameters[ParameterClusterName] = cloudformation.Parameter{ + Type: "String", + Description: "Name of the ECS cluster to deploy to (optional)", + } + + template.Parameters[ParameterVPCId] = cloudformation.Parameter{ Type: "AWS::EC2::VPC::Id", Description: "ID of the VPC", } @@ -31,17 +45,32 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Description: "The list of SubnetIds, for at least two Availability Zones in the region in your VPC", } */ - template.Parameters["Subnet1Id"] = cloudformation.Parameter{ + template.Parameters[ParameterSubnet1Id] = cloudformation.Parameter{ Type: "AWS::EC2::Subnet::Id", Description: "SubnetId,for Availability Zone 1 in the region in your VPC", } - template.Parameters["Subnet2Id"] = cloudformation.Parameter{ + template.Parameters[ParameterSubnet2Id] = cloudformation.Parameter{ Type: "AWS::EC2::Subnet::Id", Description: "SubnetId,for Availability Zone 1 in the region in your VPC", } + // Create Cluster is `ParameterClusterName` parameter is not set + template.Conditions["CreateCluster"] = cloudformation.Equals("", cloudformation.Ref(ParameterClusterName)) + + template.Resources["Cluster"] = &ecs.Cluster{ + ClusterName: project.Name, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + }, + AWSCloudFormationCondition: "CreateCluster", + } + cluster := cloudformation.If("CreateCluster", cloudformation.Ref("Cluster"), cloudformation.Ref(ParameterClusterName)) + for net := range project.Networks { - name, resource := convertNetwork(project, net, cloudformation.Ref("VPCId")) + name, resource := convertNetwork(project, net, cloudformation.Ref(ParameterVPCId)) template.Resources[name] = resource } @@ -54,7 +83,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), Name: fmt.Sprintf("%s.local", project.Name), - Vpc: cloudformation.Ref("VPCId"), + Vpc: cloudformation.Ref(ParameterVPCId), } for _, service := range project.Services { @@ -117,7 +146,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } template.Resources[fmt.Sprintf("%sService", service.Name)] = &ecs.Service{ - Cluster: c.Cluster, + Cluster: cluster, DesiredCount: 1, LaunchType: ecsapi.LaunchTypeFargate, NetworkConfiguration: &ecs.Service_NetworkConfiguration{ @@ -125,8 +154,8 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err AssignPublicIp: ecsapi.AssignPublicIpEnabled, SecurityGroups: serviceSecurityGroups, Subnets: []string{ - cloudformation.Ref("Subnet1Id"), - cloudformation.Ref("Subnet2Id"), + cloudformation.Ref(ParameterSubnet1Id), + cloudformation.Ref(ParameterSubnet2Id), }, }, }, diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index e1a0f1fd..2ad74ba8 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -9,13 +9,16 @@ import ( ) func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error { - ok, err := c.api.ClusterExists(ctx, c.Cluster) - if err != nil { - return err - } - if !ok { - c.api.CreateCluster(ctx, c.Cluster) + if c.Cluster != "" { + ok, err := c.api.ClusterExists(ctx, c.Cluster) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("configured cluster %q does not exist", c.Cluster) + } } + update, err := c.api.StackExists(ctx, project.Name) if err != nil { return err @@ -45,9 +48,10 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error } parameters := map[string]string{ - "VPCId": vpc, - "Subnet1Id": subNets[0], - "Subnet2Id": subNets[1], + ParameterClusterName: c.Cluster, + ParameterVPCId: vpc, + ParameterSubnet1Id: subNets[0], + ParameterSubnet2Id: subNets[1], } err = c.api.CreateStack(ctx, project.Name, template, parameters) @@ -87,7 +91,6 @@ type upAPI interface { GetSubNets(ctx context.Context, vpcID string) ([]string, error) ClusterExists(ctx context.Context, name string) (bool, error) - CreateCluster(ctx context.Context, name string) (string, error) StackExists(ctx context.Context, name string) (bool, error) CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error } From 9dbff1eb7272acd8497b7864123186b278fea4ca Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 13 May 2020 12:14:13 +0200 Subject: [PATCH 068/205] add logs command Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 26 ++++++++++++++++++++++++++ ecs/pkg/amazon/api.go | 1 + ecs/pkg/amazon/logs.go | 13 +++++++++++++ ecs/pkg/amazon/sdk.go | 32 ++++++++++++++++++++++++++++++++ ecs/pkg/compose/api.go | 1 + 5 files changed, 73 insertions(+) create mode 100644 ecs/pkg/amazon/logs.go diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 18a5bf81..f7a07226 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -22,6 +22,7 @@ func ComposeCommand(dockerCli command.Cli) *cobra.Command { ConvertCommand(dockerCli, opts), UpCommand(dockerCli, opts), DownCommand(dockerCli, opts), + LogsCommand(dockerCli, opts), ) return cmd } @@ -119,3 +120,28 @@ func DownCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") return cmd } + +func LogsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "logs [PROJECT NAME]", + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + var name string + + if len(args) == 0 { + project, err := compose.ProjectFromOptions(projectOpts) + if err != nil { + return err + } + name = project.Name + } else { + name = args[0] + } + return client.ComposeLogs(context.Background(), name) + }), + } + return cmd +} diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index 49306185..4fa6ddc4 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -5,5 +5,6 @@ package amazon type API interface { downAPI upAPI + logsAPI secretsAPI } diff --git a/ecs/pkg/amazon/logs.go b/ecs/pkg/amazon/logs.go new file mode 100644 index 00000000..b515e25d --- /dev/null +++ b/ecs/pkg/amazon/logs.go @@ -0,0 +1,13 @@ +package amazon + +import ( + "context" +) + +func (c *client) ComposeLogs(ctx context.Context, projectName string) error { + return c.api.GetLogs(ctx, projectName) +} + +type logsAPI interface { + GetLogs(ctx context.Context, name string) error +} diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index f0fc076f..62736d92 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -3,6 +3,7 @@ package amazon import ( "context" "fmt" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -307,3 +308,34 @@ func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { _, err := s.SM.DeleteSecret(&secretsmanager.DeleteSecretInput{SecretId: &id, ForceDeleteWithoutRecovery: &force}) return err } + +func (s sdk) GetLogs(ctx context.Context, name string) error { + logGroup := fmt.Sprintf("/docker-compose/%s", name) + var startTime int64 + for { + var hasMore = true + var token *string + token = nil + for hasMore { + events, err := s.CW.FilterLogEvents(&cloudwatchlogs.FilterLogEventsInput{ + LogGroupName: aws.String(logGroup), + NextToken: token, + StartTime: aws.Int64(startTime), + }) + if err != nil { + return err + } + if events.NextToken == nil { + hasMore = false + } else { + token = events.NextToken + } + + for _, event := range events.Events { + fmt.Println(*event.Message) + startTime = *event.IngestionTime + } + } + time.Sleep(500 * time.Millisecond) + } +} diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 70de9aa5..b39afe02 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -11,6 +11,7 @@ type API interface { Convert(project *Project) (*cloudformation.Template, error) ComposeUp(ctx context.Context, project *Project) error ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error + ComposeLogs(ctx context.Context, projectName string) error CreateSecret(ctx context.Context, secret docker.Secret) (string, error) InspectSecret(ctx context.Context, id string) (docker.Secret, error) From 0492dacfee9f2c47efd984707fa7c68da2afee9c Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 13 May 2020 12:28:23 +0200 Subject: [PATCH 069/205] remove redundant var init Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/sdk.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 62736d92..b2975130 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -315,7 +315,6 @@ func (s sdk) GetLogs(ctx context.Context, name string) error { for { var hasMore = true var token *string - token = nil for hasMore { events, err := s.CW.FilterLogEvents(&cloudwatchlogs.FilterLogEventsInput{ LogGroupName: aws.String(logGroup), From 3e5b118f26359e1be6e7316c04de61da10c29323 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 13 May 2020 14:32:17 +0200 Subject: [PATCH 070/205] add GetLogs to MockAPI Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/mock/api.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index 1210a8d3..c592162b 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -6,11 +6,12 @@ package mock import ( context "context" + reflect "reflect" + cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" docker "github.com/docker/ecs-plugin/pkg/docker" gomock "github.com/golang/mock/gomock" - reflect "reflect" ) // MockAPI is a mock of API interface @@ -270,3 +271,17 @@ func (mr *MockAPIMockRecorder) WaitStackComplete(arg0, arg1, arg2 interface{}) * mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitStackComplete", reflect.TypeOf((*MockAPI)(nil).WaitStackComplete), arg0, arg1, arg2) } + +// GetLogs mocks base method +func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLogs", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetLogs mocks base method +func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1) +} From 4bbe3f15896231f65d2b7e1b77b0aeaf998fbee1 Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Wed, 13 May 2020 15:32:24 +0200 Subject: [PATCH 071/205] Add first compose to cloudformation conversion tests Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/tests/compose_command_test.go | 49 ++++++ .../simple-single-service-with-overrides.yaml | 4 + .../testdata/input/simple-single-service.yaml | 4 + .../simple-cloudformation-conversion.golden | 164 ++++++++++++++++++ ...formation-with-overrides-conversion.golden | 164 ++++++++++++++++++ 5 files changed, 385 insertions(+) create mode 100644 ecs/tests/compose_command_test.go create mode 100644 ecs/tests/testdata/input/simple-single-service-with-overrides.yaml create mode 100644 ecs/tests/testdata/input/simple-single-service.yaml create mode 100644 ecs/tests/testdata/simple/simple-cloudformation-conversion.golden create mode 100644 ecs/tests/testdata/simple/simple-cloudformation-with-overrides-conversion.golden diff --git a/ecs/tests/compose_command_test.go b/ecs/tests/compose_command_test.go new file mode 100644 index 00000000..0bdb94bc --- /dev/null +++ b/ecs/tests/compose_command_test.go @@ -0,0 +1,49 @@ +package tests + +import ( + "testing" + + "gotest.tools/v3/fs" + "gotest.tools/v3/golden" + "gotest.tools/v3/icmd" +) + +const ( + composeFileName = "compose.yaml" +) + +func TestSimpleConvert(t *testing.T) { + cmd, cleanup := dockerCli.createTestCmd() + defer cleanup() + + composeYAML := golden.Get(t, "input/simple-single-service.yaml") + tmpDir := fs.NewDir(t, t.Name(), + fs.WithFile(composeFileName, "", fs.WithBytes(composeYAML)), + ) + defer tmpDir.Remove() + + cmd.Command = dockerCli.Command("ecs", "compose", "--file="+tmpDir.Join(composeFileName), "--project-name", t.Name(), "convert") + result := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() + + expected := "simple/simple-cloudformation-conversion.golden" + golden.Assert(t, result, expected) +} + +func TestSimpleWithOverrides(t *testing.T) { + cmd, cleanup := dockerCli.createTestCmd() + defer cleanup() + + composeYAML := golden.Get(t, "input/simple-single-service.yaml") + overriddenComposeYAML := golden.Get(t, "input/simple-single-service-with-overrides.yaml") + tmpDir := fs.NewDir(t, t.Name(), + fs.WithFile(composeFileName, "", fs.WithBytes(composeYAML)), + fs.WithFile("overriddenService.yaml", "", fs.WithBytes(overriddenComposeYAML)), + ) + defer tmpDir.Remove() + cmd.Command = dockerCli.Command("ecs", "compose", "--file="+tmpDir.Join(composeFileName), "--file", + tmpDir.Join("overriddenService.yaml"), "--project-name", t.Name(), "convert") + result := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() + + expected := "simple/simple-cloudformation-with-overrides-conversion.golden" + golden.Assert(t, result, expected) +} diff --git a/ecs/tests/testdata/input/simple-single-service-with-overrides.yaml b/ecs/tests/testdata/input/simple-single-service-with-overrides.yaml new file mode 100644 index 00000000..3dc8a0b6 --- /dev/null +++ b/ecs/tests/testdata/input/simple-single-service-with-overrides.yaml @@ -0,0 +1,4 @@ +version: "3" +services: + simple: + image: haproxy diff --git a/ecs/tests/testdata/input/simple-single-service.yaml b/ecs/tests/testdata/input/simple-single-service.yaml new file mode 100644 index 00000000..4b3f9af2 --- /dev/null +++ b/ecs/tests/testdata/input/simple-single-service.yaml @@ -0,0 +1,4 @@ +version: "3" +services: + simple: + image: nginx \ No newline at end of file diff --git a/ecs/tests/testdata/simple/simple-cloudformation-conversion.golden b/ecs/tests/testdata/simple/simple-cloudformation-conversion.golden new file mode 100644 index 00000000..51a528b3 --- /dev/null +++ b/ecs/tests/testdata/simple/simple-cloudformation-conversion.golden @@ -0,0 +1,164 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Parameters": { + "Subnet1Id": { + "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", + "Type": "AWS::EC2::Subnet::Id" + }, + "Subnet2Id": { + "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", + "Type": "AWS::EC2::Subnet::Id" + }, + "VPCId": { + "Description": "ID of the VPC", + "Type": "AWS::EC2::VPC::Id" + } + }, + "Resources": { + "CloudMap": { + "Properties": { + "Description": "Service Map for Docker Compose project TestSimpleConvert", + "Name": "TestSimpleConvert.local", + "Vpc": { + "Ref": "VPCId" + } + }, + "Type": "AWS::ServiceDiscovery::PrivateDnsNamespace" + }, + "LogGroup": { + "Properties": { + "LogGroupName": "/docker-compose/TestSimpleConvert" + }, + "Type": "AWS::Logs::LogGroup" + }, + "simpleService": { + "Properties": { + "Cluster": "TestCluster", + "DesiredCount": 1, + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "AssignPublicIp": "ENABLED", + "Subnets": [ + { + "Ref": "Subnet1Id" + }, + { + "Ref": "Subnet2Id" + } + ] + } + }, + "SchedulingStrategy": "REPLICA", + "ServiceName": "simple", + "ServiceRegistries": [ + { + "RegistryArn": { + "Fn::GetAtt": [ + "simpleServiceDiscoveryEntry", + "Arn" + ] + } + } + ], + "TaskDefinition": { + "Ref": "simpleTaskDefinition" + } + }, + "Type": "AWS::ECS::Service" + }, + "simpleServiceDiscoveryEntry": { + "Properties": { + "Description": "\"simple\" service discovery entry in Cloud Map", + "DnsConfig": { + "DnsRecords": [ + { + "TTL": 300, + "Type": "A" + } + ], + "RoutingPolicy": "MULTIVALUE" + }, + "Name": "simple", + "NamespaceId": { + "Ref": "CloudMap" + } + }, + "Type": "AWS::ServiceDiscovery::Service" + }, + "simpleTaskDefinition": { + "Properties": { + "ContainerDefinitions": [ + { + "Environment": [ + { + "Name": "LOCALDOMAIN", + "Value": { + "Fn::Join": [ + "", + [ + { + "Ref": "AWS::Region" + }, + ".compute.internal", + " TestSimpleConvert.local" + ] + ] + } + } + ], + "Essential": true, + "Image": "docker.io/library/nginx", + "LinuxParameters": {}, + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "LogGroup" + }, + "awslogs-region": { + "Ref": "AWS::Region" + }, + "awslogs-stream-prefix": "simple" + } + }, + "Name": "simple" + } + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Ref": "simpleTaskExecutionRole" + }, + "Family": "TestSimpleConvert-simple", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE" + ] + }, + "Type": "AWS::ECS::TaskDefinition" + }, + "simpleTaskExecutionRole": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + ] + }, + "Type": "AWS::IAM::Role" + } + } +} diff --git a/ecs/tests/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/tests/testdata/simple/simple-cloudformation-with-overrides-conversion.golden new file mode 100644 index 00000000..2e8f5a4a --- /dev/null +++ b/ecs/tests/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -0,0 +1,164 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Parameters": { + "Subnet1Id": { + "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", + "Type": "AWS::EC2::Subnet::Id" + }, + "Subnet2Id": { + "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", + "Type": "AWS::EC2::Subnet::Id" + }, + "VPCId": { + "Description": "ID of the VPC", + "Type": "AWS::EC2::VPC::Id" + } + }, + "Resources": { + "CloudMap": { + "Properties": { + "Description": "Service Map for Docker Compose project TestSimpleWithOverrides", + "Name": "TestSimpleWithOverrides.local", + "Vpc": { + "Ref": "VPCId" + } + }, + "Type": "AWS::ServiceDiscovery::PrivateDnsNamespace" + }, + "LogGroup": { + "Properties": { + "LogGroupName": "/docker-compose/TestSimpleWithOverrides" + }, + "Type": "AWS::Logs::LogGroup" + }, + "simpleService": { + "Properties": { + "Cluster": "TestCluster", + "DesiredCount": 1, + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "AssignPublicIp": "ENABLED", + "Subnets": [ + { + "Ref": "Subnet1Id" + }, + { + "Ref": "Subnet2Id" + } + ] + } + }, + "SchedulingStrategy": "REPLICA", + "ServiceName": "simple", + "ServiceRegistries": [ + { + "RegistryArn": { + "Fn::GetAtt": [ + "simpleServiceDiscoveryEntry", + "Arn" + ] + } + } + ], + "TaskDefinition": { + "Ref": "simpleTaskDefinition" + } + }, + "Type": "AWS::ECS::Service" + }, + "simpleServiceDiscoveryEntry": { + "Properties": { + "Description": "\"simple\" service discovery entry in Cloud Map", + "DnsConfig": { + "DnsRecords": [ + { + "TTL": 300, + "Type": "A" + } + ], + "RoutingPolicy": "MULTIVALUE" + }, + "Name": "simple", + "NamespaceId": { + "Ref": "CloudMap" + } + }, + "Type": "AWS::ServiceDiscovery::Service" + }, + "simpleTaskDefinition": { + "Properties": { + "ContainerDefinitions": [ + { + "Environment": [ + { + "Name": "LOCALDOMAIN", + "Value": { + "Fn::Join": [ + "", + [ + { + "Ref": "AWS::Region" + }, + ".compute.internal", + " TestSimpleWithOverrides.local" + ] + ] + } + } + ], + "Essential": true, + "Image": "docker.io/library/haproxy", + "LinuxParameters": {}, + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "LogGroup" + }, + "awslogs-region": { + "Ref": "AWS::Region" + }, + "awslogs-stream-prefix": "simple" + } + }, + "Name": "simple" + } + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Ref": "simpleTaskExecutionRole" + }, + "Family": "TestSimpleWithOverrides-simple", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE" + ] + }, + "Type": "AWS::ECS::TaskDefinition" + }, + "simpleTaskExecutionRole": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + ] + }, + "Type": "AWS::IAM::Role" + } + } +} From 07a57469dbf9fa8b929554203782d0b7ac889b4f Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Wed, 13 May 2020 18:02:52 +0200 Subject: [PATCH 072/205] Add unit tests version of migration tests instead of e2e one Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/go.sum | 1 + ecs/pkg/amazon/cloudformation_test.go | 49 +++++++++++++++++++ .../simple-single-service-with-overrides.yaml | 0 .../testdata/input/simple-single-service.yaml | 0 .../simple-cloudformation-conversion.golden | 0 ...formation-with-overrides-conversion.golden | 0 ecs/pkg/compose/opts.go | 4 +- ecs/pkg/compose/project.go | 2 +- ecs/pkg/compose/project_test.go | 8 +-- ecs/tests/compose_command_test.go | 49 ------------------- 10 files changed, 57 insertions(+), 56 deletions(-) create mode 100644 ecs/pkg/amazon/cloudformation_test.go rename ecs/{tests => pkg/amazon}/testdata/input/simple-single-service-with-overrides.yaml (100%) rename ecs/{tests => pkg/amazon}/testdata/input/simple-single-service.yaml (100%) rename ecs/{tests => pkg/amazon}/testdata/simple/simple-cloudformation-conversion.golden (100%) rename ecs/{tests => pkg/amazon}/testdata/simple/simple-cloudformation-with-overrides-conversion.golden (100%) delete mode 100644 ecs/tests/compose_command_test.go diff --git a/ecs/go.sum b/ecs/go.sum index 5ca9a4bf..9088eaf2 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -213,6 +213,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= diff --git a/ecs/pkg/amazon/cloudformation_test.go b/ecs/pkg/amazon/cloudformation_test.go new file mode 100644 index 00000000..e4507338 --- /dev/null +++ b/ecs/pkg/amazon/cloudformation_test.go @@ -0,0 +1,49 @@ +package amazon + +import ( + "fmt" + "testing" + + "github.com/docker/ecs-plugin/pkg/compose" + "gotest.tools/v3/golden" +) + +func TestSimpleConvert(t *testing.T) { + options := compose.ProjectOptions{ + Name: t.Name(), + ConfigPaths: []string{"testdata/input/simple-single-service.yaml"}, + } + result := convertResultAsString(t, options, "TestCluster") + expected := "simple/simple-cloudformation-conversion.golden" + golden.Assert(t, result, expected) +} + +func TestSimpleWithOverrides(t *testing.T) { + options := compose.ProjectOptions{ + Name: t.Name(), + ConfigPaths: []string{"testdata/input/simple-single-service.yaml", "testdata/input/simple-single-service-with-overrides.yaml"}, + } + result := convertResultAsString(t, options, "TestCluster") + expected := "simple/simple-cloudformation-with-overrides-conversion.golden" + golden.Assert(t, result, expected) +} + +func convertResultAsString(t *testing.T, options compose.ProjectOptions, clusterName string) string { + project, err := compose.ProjectFromOptions(&options) + if err != nil { + t.Error(err) + } + client, err := NewClient("", clusterName, "") + if err != nil { + t.Error(err) + } + result, err := client.Convert(project) + if err != nil { + t.Error(err) + } + resultAsJSON, err := result.JSON() + if err != nil { + t.Error(err) + } + return fmt.Sprintf("%s\n", string(resultAsJSON)) +} diff --git a/ecs/tests/testdata/input/simple-single-service-with-overrides.yaml b/ecs/pkg/amazon/testdata/input/simple-single-service-with-overrides.yaml similarity index 100% rename from ecs/tests/testdata/input/simple-single-service-with-overrides.yaml rename to ecs/pkg/amazon/testdata/input/simple-single-service-with-overrides.yaml diff --git a/ecs/tests/testdata/input/simple-single-service.yaml b/ecs/pkg/amazon/testdata/input/simple-single-service.yaml similarity index 100% rename from ecs/tests/testdata/input/simple-single-service.yaml rename to ecs/pkg/amazon/testdata/input/simple-single-service.yaml diff --git a/ecs/tests/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden similarity index 100% rename from ecs/tests/testdata/simple/simple-cloudformation-conversion.golden rename to ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden diff --git a/ecs/tests/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden similarity index 100% rename from ecs/tests/testdata/simple/simple-cloudformation-with-overrides-conversion.golden rename to ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden diff --git a/ecs/pkg/compose/opts.go b/ecs/pkg/compose/opts.go index 9e9bdeee..d2fef0c9 100644 --- a/ecs/pkg/compose/opts.go +++ b/ecs/pkg/compose/opts.go @@ -7,12 +7,12 @@ import ( type ProjectOptions struct { ConfigPaths []string - name string + Name string } func (o *ProjectOptions) AddFlags(flags *pflag.FlagSet) { flags.StringArrayVarP(&o.ConfigPaths, "file", "f", nil, "Specify an alternate compose file") - flags.StringVarP(&o.name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") + flags.StringVarP(&o.Name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") } type ProjectFunc func(project *Project, args []string) error diff --git a/ecs/pkg/compose/project.go b/ecs/pkg/compose/project.go index e17e2252..5c244b2c 100644 --- a/ecs/pkg/compose/project.go +++ b/ecs/pkg/compose/project.go @@ -40,7 +40,7 @@ func ProjectFromOptions(options *ProjectOptions) (*Project, error) { return nil, err } - name := options.name + name := options.Name if name == "" { name = os.Getenv("COMPOSE_PROJECT_NAME") } diff --git a/ecs/pkg/compose/project_test.go b/ecs/pkg/compose/project_test.go index 2906c9d2..733f34f0 100644 --- a/ecs/pkg/compose/project_test.go +++ b/ecs/pkg/compose/project_test.go @@ -9,14 +9,14 @@ import ( func Test_project_name(t *testing.T) { p, err := ProjectFromOptions(&ProjectOptions{ - name: "my_project", + Name: "my_project", ConfigPaths: []string{"testdata/simple/compose.yaml"}, }) assert.NilError(t, err) assert.Equal(t, p.Name, "my_project") p, err = ProjectFromOptions(&ProjectOptions{ - name: "", + Name: "", ConfigPaths: []string{"testdata/simple/compose.yaml"}, }) assert.NilError(t, err) @@ -24,7 +24,7 @@ func Test_project_name(t *testing.T) { os.Setenv("COMPOSE_PROJECT_NAME", "my_project_from_env") p, err = ProjectFromOptions(&ProjectOptions{ - name: "", + Name: "", ConfigPaths: []string{"testdata/simple/compose.yaml"}, }) assert.NilError(t, err) @@ -33,7 +33,7 @@ func Test_project_name(t *testing.T) { func Test_project_from_set_of_files(t *testing.T) { p, err := ProjectFromOptions(&ProjectOptions{ - name: "my_project", + Name: "my_project", ConfigPaths: []string{ "testdata/simple/compose.yaml", "testdata/simple/compose-with-overrides.yaml", diff --git a/ecs/tests/compose_command_test.go b/ecs/tests/compose_command_test.go deleted file mode 100644 index 0bdb94bc..00000000 --- a/ecs/tests/compose_command_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package tests - -import ( - "testing" - - "gotest.tools/v3/fs" - "gotest.tools/v3/golden" - "gotest.tools/v3/icmd" -) - -const ( - composeFileName = "compose.yaml" -) - -func TestSimpleConvert(t *testing.T) { - cmd, cleanup := dockerCli.createTestCmd() - defer cleanup() - - composeYAML := golden.Get(t, "input/simple-single-service.yaml") - tmpDir := fs.NewDir(t, t.Name(), - fs.WithFile(composeFileName, "", fs.WithBytes(composeYAML)), - ) - defer tmpDir.Remove() - - cmd.Command = dockerCli.Command("ecs", "compose", "--file="+tmpDir.Join(composeFileName), "--project-name", t.Name(), "convert") - result := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() - - expected := "simple/simple-cloudformation-conversion.golden" - golden.Assert(t, result, expected) -} - -func TestSimpleWithOverrides(t *testing.T) { - cmd, cleanup := dockerCli.createTestCmd() - defer cleanup() - - composeYAML := golden.Get(t, "input/simple-single-service.yaml") - overriddenComposeYAML := golden.Get(t, "input/simple-single-service-with-overrides.yaml") - tmpDir := fs.NewDir(t, t.Name(), - fs.WithFile(composeFileName, "", fs.WithBytes(composeYAML)), - fs.WithFile("overriddenService.yaml", "", fs.WithBytes(overriddenComposeYAML)), - ) - defer tmpDir.Remove() - cmd.Command = dockerCli.Command("ecs", "compose", "--file="+tmpDir.Join(composeFileName), "--file", - tmpDir.Join("overriddenService.yaml"), "--project-name", t.Name(), "convert") - result := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() - - expected := "simple/simple-cloudformation-with-overrides-conversion.golden" - golden.Assert(t, result, expected) -} From 8cd4a6fe9b257f5658e92b1dd2fc040b75b46d9c Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 13 May 2020 22:09:20 +0200 Subject: [PATCH 073/205] Fix golden files after rebase Signed-off-by: Nicolas De Loof --- .../simple-cloudformation-conversion.golden | 51 ++++++++++++++++--- ...formation-with-overrides-conversion.golden | 51 ++++++++++++++++--- 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 51a528b3..d0e632d9 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -1,15 +1,29 @@ { "AWSTemplateFormatVersion": "2010-09-09", + "Conditions": { + "CreateCluster": { + "Fn::Equals": [ + "", + { + "Ref": "ParameterClusterName" + } + ] + } + }, "Parameters": { - "Subnet1Id": { + "ParameterClusterName": { + "Description": "Name of the ECS cluster to deploy to (optional)", + "Type": "String" + }, + "ParameterSubnet1Id": { "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" }, - "Subnet2Id": { + "ParameterSubnet2Id": { "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" }, - "VPCId": { + "ParameterVPCId": { "Description": "ID of the VPC", "Type": "AWS::EC2::VPC::Id" } @@ -20,11 +34,24 @@ "Description": "Service Map for Docker Compose project TestSimpleConvert", "Name": "TestSimpleConvert.local", "Vpc": { - "Ref": "VPCId" + "Ref": "ParameterVPCId" } }, "Type": "AWS::ServiceDiscovery::PrivateDnsNamespace" }, + "Cluster": { + "Condition": "CreateCluster", + "Properties": { + "ClusterName": "TestSimpleConvert", + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleConvert" + } + ] + }, + "Type": "AWS::ECS::Cluster" + }, "LogGroup": { "Properties": { "LogGroupName": "/docker-compose/TestSimpleConvert" @@ -33,7 +60,17 @@ }, "simpleService": { "Properties": { - "Cluster": "TestCluster", + "Cluster": { + "Fn::If": [ + "CreateCluster", + { + "Ref": "Cluster" + }, + { + "Ref": "ParameterClusterName" + } + ] + }, "DesiredCount": 1, "LaunchType": "FARGATE", "NetworkConfiguration": { @@ -41,10 +78,10 @@ "AssignPublicIp": "ENABLED", "Subnets": [ { - "Ref": "Subnet1Id" + "Ref": "ParameterSubnet1Id" }, { - "Ref": "Subnet2Id" + "Ref": "ParameterSubnet2Id" } ] } diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 2e8f5a4a..5e641af5 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -1,15 +1,29 @@ { "AWSTemplateFormatVersion": "2010-09-09", + "Conditions": { + "CreateCluster": { + "Fn::Equals": [ + "", + { + "Ref": "ParameterClusterName" + } + ] + } + }, "Parameters": { - "Subnet1Id": { + "ParameterClusterName": { + "Description": "Name of the ECS cluster to deploy to (optional)", + "Type": "String" + }, + "ParameterSubnet1Id": { "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" }, - "Subnet2Id": { + "ParameterSubnet2Id": { "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" }, - "VPCId": { + "ParameterVPCId": { "Description": "ID of the VPC", "Type": "AWS::EC2::VPC::Id" } @@ -20,11 +34,24 @@ "Description": "Service Map for Docker Compose project TestSimpleWithOverrides", "Name": "TestSimpleWithOverrides.local", "Vpc": { - "Ref": "VPCId" + "Ref": "ParameterVPCId" } }, "Type": "AWS::ServiceDiscovery::PrivateDnsNamespace" }, + "Cluster": { + "Condition": "CreateCluster", + "Properties": { + "ClusterName": "TestSimpleWithOverrides", + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleWithOverrides" + } + ] + }, + "Type": "AWS::ECS::Cluster" + }, "LogGroup": { "Properties": { "LogGroupName": "/docker-compose/TestSimpleWithOverrides" @@ -33,7 +60,17 @@ }, "simpleService": { "Properties": { - "Cluster": "TestCluster", + "Cluster": { + "Fn::If": [ + "CreateCluster", + { + "Ref": "Cluster" + }, + { + "Ref": "ParameterClusterName" + } + ] + }, "DesiredCount": 1, "LaunchType": "FARGATE", "NetworkConfiguration": { @@ -41,10 +78,10 @@ "AssignPublicIp": "ENABLED", "Subnets": [ { - "Ref": "Subnet1Id" + "Ref": "ParameterSubnet1Id" }, { - "Ref": "Subnet2Id" + "Ref": "ParameterSubnet2Id" } ] } From ae4dc2e0db7ff27b52ce0c21e36948f7d2b2e2d9 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 18 May 2020 10:53:07 +0200 Subject: [PATCH 074/205] Reject compose file with unsupported features Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 5 +++ ecs/pkg/amazon/cloudformation_test.go | 44 +++++++++---------- .../amazon/testdata/invalid_network_mode.yaml | 5 +++ ecs/pkg/amazon/up.go | 5 --- ecs/pkg/amazon/validate.go | 8 +++- ecs/pkg/amazon/validate_test.go | 13 ++++++ 6 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 ecs/pkg/amazon/testdata/invalid_network_mode.yaml create mode 100644 ecs/pkg/amazon/validate_test.go diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index d14b5faf..93ed2e09 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -26,6 +26,11 @@ const ( // Convert a compose project into a CloudFormation template func (c client) Convert(project *compose.Project) (*cloudformation.Template, error) { + err := Validate(project) + if err != nil { + return nil, err + } + template := cloudformation.NewTemplate() template.Parameters[ParameterClusterName] = cloudformation.Parameter{ diff --git a/ecs/pkg/amazon/cloudformation_test.go b/ecs/pkg/amazon/cloudformation_test.go index e4507338..456efbd0 100644 --- a/ecs/pkg/amazon/cloudformation_test.go +++ b/ecs/pkg/amazon/cloudformation_test.go @@ -4,46 +4,42 @@ import ( "fmt" "testing" + "gotest.tools/assert" + "github.com/docker/ecs-plugin/pkg/compose" "gotest.tools/v3/golden" ) func TestSimpleConvert(t *testing.T) { - options := compose.ProjectOptions{ - Name: t.Name(), - ConfigPaths: []string{"testdata/input/simple-single-service.yaml"}, - } - result := convertResultAsString(t, options, "TestCluster") + project := load(t, "testdata/input/simple-single-service.yaml") + result := convertResultAsString(t, project, "TestCluster") expected := "simple/simple-cloudformation-conversion.golden" golden.Assert(t, result, expected) } func TestSimpleWithOverrides(t *testing.T) { - options := compose.ProjectOptions{ - Name: t.Name(), - ConfigPaths: []string{"testdata/input/simple-single-service.yaml", "testdata/input/simple-single-service-with-overrides.yaml"}, - } - result := convertResultAsString(t, options, "TestCluster") + project := load(t, "testdata/input/simple-single-service.yaml", "testdata/input/simple-single-service-with-overrides.yaml") + result := convertResultAsString(t, project, "TestCluster") expected := "simple/simple-cloudformation-with-overrides-conversion.golden" golden.Assert(t, result, expected) } -func convertResultAsString(t *testing.T, options compose.ProjectOptions, clusterName string) string { - project, err := compose.ProjectFromOptions(&options) - if err != nil { - t.Error(err) - } +func convertResultAsString(t *testing.T, project *compose.Project, clusterName string) string { client, err := NewClient("", clusterName, "") - if err != nil { - t.Error(err) - } + assert.NilError(t, err) result, err := client.Convert(project) - if err != nil { - t.Error(err) - } + assert.NilError(t, err) resultAsJSON, err := result.JSON() - if err != nil { - t.Error(err) - } + assert.NilError(t, err) return fmt.Sprintf("%s\n", string(resultAsJSON)) } + +func load(t *testing.T, paths ...string) *compose.Project { + options := compose.ProjectOptions{ + Name: t.Name(), + ConfigPaths: paths, + } + project, err := compose.ProjectFromOptions(&options) + assert.NilError(t, err) + return project +} diff --git a/ecs/pkg/amazon/testdata/invalid_network_mode.yaml b/ecs/pkg/amazon/testdata/invalid_network_mode.yaml new file mode 100644 index 00000000..ce8ed8ad --- /dev/null +++ b/ecs/pkg/amazon/testdata/invalid_network_mode.yaml @@ -0,0 +1,5 @@ +version: "3" +services: + simple: + image: nginx + network_mode: bridge \ No newline at end of file diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 2ad74ba8..e3297249 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -27,11 +27,6 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") } - err = c.Validate(project) - if err != nil { - return err - } - template, err := c.Convert(project) if err != nil { return err diff --git a/ecs/pkg/amazon/validate.go b/ecs/pkg/amazon/validate.go index 551dc7bc..4e7918fc 100644 --- a/ecs/pkg/amazon/validate.go +++ b/ecs/pkg/amazon/validate.go @@ -1,12 +1,14 @@ package amazon import ( + "fmt" + "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" ) // Validate check the compose model do not use unsupported features and inject sane defaults for ECS deployment -func (c *client) Validate(project *compose.Project) error { +func Validate(project *compose.Project) error { if len(project.Networks) == 0 { // Compose application model implies a default network if none is explicitly set. // FIXME move this to compose-go @@ -22,6 +24,10 @@ func (c *client) Validate(project *compose.Project) error { service.Networks = map[string]*types.ServiceNetworkConfig{"default": nil} project.Services[i] = service } + + if service.NetworkMode != "" && service.NetworkMode != "awsvpc" { + return fmt.Errorf("ECS do not support NetworkMode %q", service.NetworkMode) + } } // Here we can check for incompatible attributes, inject sane defaults, etc diff --git a/ecs/pkg/amazon/validate_test.go b/ecs/pkg/amazon/validate_test.go new file mode 100644 index 00000000..e0cdd650 --- /dev/null +++ b/ecs/pkg/amazon/validate_test.go @@ -0,0 +1,13 @@ +package amazon + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestInvalidNetworkMode(t *testing.T) { + project := load(t, "testdata/invalid_network_mode.yaml") + err := Validate(project) + assert.Error(t, err, "ECS do not support NetworkMode \"bridge\"") +} From 6798ad12451823ef8cc5113203249e108800f069 Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Mon, 18 May 2020 11:36:42 +0200 Subject: [PATCH 075/205] Add security group declaration in cloudformation conversion tests Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- .../simple-cloudformation-conversion.golden | 25 +++++++++++++++++++ ...formation-with-overrides-conversion.golden | 25 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index d0e632d9..0910aaf3 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -58,6 +58,26 @@ }, "Type": "AWS::Logs::LogGroup" }, + "TestSimpleConvertDefaultNetwork": { + "Properties": { + "GroupDescription": "TestSimpleConvert default Security Group", + "GroupName": "TestSimpleConvertDefaultNetwork", + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleConvert" + }, + { + "Key": "com.docker.compose.network", + "Value": "default" + } + ], + "VpcId": { + "Ref": "ParameterVPCId" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, "simpleService": { "Properties": { "Cluster": { @@ -76,6 +96,11 @@ "NetworkConfiguration": { "AwsvpcConfiguration": { "AssignPublicIp": "ENABLED", + "SecurityGroups": [ + { + "Ref": "TestSimpleConvertDefaultNetwork" + } + ], "Subnets": [ { "Ref": "ParameterSubnet1Id" diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 5e641af5..7c413438 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -58,6 +58,26 @@ }, "Type": "AWS::Logs::LogGroup" }, + "TestSimpleWithOverridesDefaultNetwork": { + "Properties": { + "GroupDescription": "TestSimpleWithOverrides default Security Group", + "GroupName": "TestSimpleWithOverridesDefaultNetwork", + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleWithOverrides" + }, + { + "Key": "com.docker.compose.network", + "Value": "default" + } + ], + "VpcId": { + "Ref": "ParameterVPCId" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, "simpleService": { "Properties": { "Cluster": { @@ -76,6 +96,11 @@ "NetworkConfiguration": { "AwsvpcConfiguration": { "AssignPublicIp": "ENABLED", + "SecurityGroups": [ + { + "Ref": "TestSimpleWithOverridesDefaultNetwork" + } + ], "Subnets": [ { "Ref": "ParameterSubnet1Id" From a5a925173c9c12364d8bf23fd716500dd930db21 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 18 May 2020 16:01:00 +0200 Subject: [PATCH 076/205] SDK methods to query service tasks and retrieve public IP Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/sdk.go | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index b2975130..2df26efd 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -338,3 +338,55 @@ func (s sdk) GetLogs(ctx context.Context, name string) error { time.Sleep(500 * time.Millisecond) } } + +func (s sdk) GetTasks(ctx context.Context, cluster string, name string) ([]string, error) { + tasks, err := s.ECS.ListTasksWithContext(ctx, &ecs.ListTasksInput{ + Cluster: aws.String(cluster), + ServiceName: aws.String(name), + }) + if err != nil { + return nil, err + } + arns := []string{} + for _, arn := range tasks.TaskArns { + arns = append(arns, *arn) + } + return arns, nil +} + +func (s sdk) GetNetworkInterfaces(ctx context.Context, cluster string, arns ...string) ([]string, error) { + tasks, err := s.ECS.DescribeTasksWithContext(ctx, &ecs.DescribeTasksInput{ + Cluster: aws.String(cluster), + Tasks: aws.StringSlice(arns), + }) + if err != nil { + return nil, err + } + interfaces := []string{} + for _, task := range tasks.Tasks { + for _, attachement := range task.Attachments { + if *attachement.Type == "ElasticNetworkInterface" { + for _, pair := range attachement.Details { + if *pair.Name == "networkInterfaceId" { + interfaces = append(interfaces, *pair.Value) + } + } + } + } + } + return interfaces, nil +} + +func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) ([]string, error) { + desc, err := s.EC2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ + NetworkInterfaceIds: aws.StringSlice(interfaces), + }) + if err != nil { + return nil, err + } + publicIPs := []string{} + for _, interf := range desc.NetworkInterfaces { + publicIPs = append(publicIPs, *interf.Association.PublicIp) + } + return publicIPs, nil +} From 08bd18231dd30f7564fcf5dbb205b97ea0a283db Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 19 May 2020 15:27:11 +0200 Subject: [PATCH 077/205] Introduce `Normalize` and `Check` in compose model lifecycle Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/check.go | 42 ++++ .../{validate_test.go => check_test.go} | 4 +- ecs/pkg/amazon/cloudformation.go | 8 +- ecs/pkg/amazon/compatibility.go | 179 ++++++++++++++++++ ecs/pkg/amazon/convert.go | 27 ++- ecs/pkg/amazon/validate.go | 35 ---- ecs/pkg/compose/normalize.go | 53 ++++++ ecs/pkg/compose/project.go | 5 + 8 files changed, 311 insertions(+), 42 deletions(-) create mode 100644 ecs/pkg/amazon/check.go rename ecs/pkg/amazon/{validate_test.go => check_test.go} (63%) create mode 100644 ecs/pkg/amazon/compatibility.go delete mode 100644 ecs/pkg/amazon/validate.go create mode 100644 ecs/pkg/compose/normalize.go diff --git a/ecs/pkg/amazon/check.go b/ecs/pkg/amazon/check.go new file mode 100644 index 00000000..d249afe4 --- /dev/null +++ b/ecs/pkg/amazon/check.go @@ -0,0 +1,42 @@ +package amazon + +import ( + "github.com/compose-spec/compose-go/types" + "github.com/docker/ecs-plugin/pkg/compose" +) + +type Warning string +type Warnings []string + +type CompatibilityChecker interface { + CheckService(service *types.ServiceConfig) + CheckCapAdd(service *types.ServiceConfig) + CheckDependsOn(service *types.ServiceConfig) + CheckDNS(service *types.ServiceConfig) + CheckDNSOpts(service *types.ServiceConfig) + CheckDNSSearch(service *types.ServiceConfig) + CheckDomainName(service *types.ServiceConfig) + CheckExtraHosts(service *types.ServiceConfig) + CheckHostname(service *types.ServiceConfig) + CheckIpc(service *types.ServiceConfig) + CheckLabels(service *types.ServiceConfig) + CheckLinks(service *types.ServiceConfig) + CheckLogging(service *types.ServiceConfig) + CheckMacAddress(service *types.ServiceConfig) + CheckNetworkMode(service *types.ServiceConfig) + CheckPid(service *types.ServiceConfig) + CheckSysctls(service *types.ServiceConfig) + CheckTmpfs(service *types.ServiceConfig) + CheckUserNSMode(service *types.ServiceConfig) + Errors() []error +} + +// Check the compose model do not use unsupported features and inject sane defaults for ECS deployment +func Check(project *compose.Project) []error { + c := FargateCompatibilityChecker{} + for i, service := range project.Services { + c.CheckService(&service) + project.Services[i] = service + } + return c.errors +} diff --git a/ecs/pkg/amazon/validate_test.go b/ecs/pkg/amazon/check_test.go similarity index 63% rename from ecs/pkg/amazon/validate_test.go rename to ecs/pkg/amazon/check_test.go index e0cdd650..3f185954 100644 --- a/ecs/pkg/amazon/validate_test.go +++ b/ecs/pkg/amazon/check_test.go @@ -8,6 +8,6 @@ import ( func TestInvalidNetworkMode(t *testing.T) { project := load(t, "testdata/invalid_network_mode.yaml") - err := Validate(project) - assert.Error(t, err, "ECS do not support NetworkMode \"bridge\"") + err := Check(project) + assert.Error(t, err[0], "'network_mode' \"bridge\" is not supported") } diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 93ed2e09..2eb6f1df 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" + "github.com/sirupsen/logrus" + cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" ecsapi "github.com/aws/aws-sdk-go/service/ecs" @@ -26,9 +28,9 @@ const ( // Convert a compose project into a CloudFormation template func (c client) Convert(project *compose.Project) (*cloudformation.Template, error) { - err := Validate(project) - if err != nil { - return nil, err + warnings := Check(project) + for _, w := range warnings { + logrus.Warn(w) } template := cloudformation.NewTemplate() diff --git a/ecs/pkg/amazon/compatibility.go b/ecs/pkg/amazon/compatibility.go new file mode 100644 index 00000000..6049a4d4 --- /dev/null +++ b/ecs/pkg/amazon/compatibility.go @@ -0,0 +1,179 @@ +package amazon + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/service/ecs" + "github.com/compose-spec/compose-go/types" +) + +type FargateCompatibilityChecker struct { + errors []error +} + +func (c *FargateCompatibilityChecker) error(message string, args ...interface{}) { + c.errors = append(c.errors, fmt.Errorf(message, args...)) +} + +func (c *FargateCompatibilityChecker) Errors() []error { + return c.errors +} + +func (c *FargateCompatibilityChecker) CheckService(service *types.ServiceConfig) { + c.CheckCapAdd(service) + c.CheckDependsOn(service) + c.CheckDNS(service) + c.CheckDNSOpts(service) + c.CheckDNSSearch(service) + c.CheckDomainName(service) + c.CheckExtraHosts(service) + c.CheckHostname(service) + c.CheckIpc(service) + c.CheckLabels(service) + c.CheckLinks(service) + c.CheckLogging(service) + c.CheckMacAddress(service) + c.CheckNetworkMode(service) + c.CheckPid(service) + c.CheckSysctls(service) + c.CheckTmpfs(service) + c.CheckUserNSMode(service) +} + +func (c *FargateCompatibilityChecker) CheckNetworkMode(service *types.ServiceConfig) { + if service.NetworkMode != "" && service.NetworkMode != ecs.NetworkModeAwsvpc { + c.error("'network_mode' %q is not supported", service.NetworkMode) + } + service.NetworkMode = ecs.NetworkModeAwsvpc +} + +func (c *FargateCompatibilityChecker) CheckDependsOn(service *types.ServiceConfig) { + if len(service.DependsOn) != 0 { + c.error("'depends_on' is not supported") + service.DependsOn = nil + } +} + +func (c *FargateCompatibilityChecker) CheckLinks(service *types.ServiceConfig) { + if len(service.Links) != 0 { + c.error("'links' is not supported") + service.Links = nil + } +} + +func (c *FargateCompatibilityChecker) CheckLogging(service *types.ServiceConfig) { + c.CheckLoggingDriver(service) +} + +func (c *FargateCompatibilityChecker) CheckLoggingDriver(service *types.ServiceConfig) { + if service.LogDriver != "" && service.LogDriver != ecs.LogDriverAwslogs { + c.error("'log_driver' %q is not supported", service.LogDriver) + service.LogDriver = ecs.LogDriverAwslogs + } +} + +func (c *FargateCompatibilityChecker) CheckPid(service *types.ServiceConfig) { + if service.Pid != "" { + c.error("'pid' is not supported") + service.Pid = "" + } +} + +func (c *FargateCompatibilityChecker) CheckUserNSMode(service *types.ServiceConfig) { + if service.UserNSMode != "" { + c.error("'userns_mode' is not supported") + service.UserNSMode = "" + } +} + +func (c *FargateCompatibilityChecker) CheckIpc(service *types.ServiceConfig) { + if service.Ipc != "" { + c.error("'ipc' is not supported") + service.Ipc = "" + } +} + +func (c *FargateCompatibilityChecker) CheckMacAddress(service *types.ServiceConfig) { + if service.MacAddress != "" { + c.error("'mac_address' is not supported") + service.MacAddress = "" + } +} + +func (c *FargateCompatibilityChecker) CheckHostname(service *types.ServiceConfig) { + if service.Hostname != "" { + c.error("'hostname' is not supported") + service.Hostname = "" + } +} + +func (c *FargateCompatibilityChecker) CheckDomainName(service *types.ServiceConfig) { + if service.DomainName != "" { + c.error("'domainname' is not supported") + service.DomainName = "" + } +} + +func (c *FargateCompatibilityChecker) CheckDNSSearch(service *types.ServiceConfig) { + if len(service.DNSSearch) > 0 { + c.error("'dns_search' is not supported") + service.DNSSearch = nil + } +} + +func (c *FargateCompatibilityChecker) CheckDNS(service *types.ServiceConfig) { + if len(service.DNS) > 0 { + c.error("'dns' is not supported") + service.DNS = nil + } +} + +func (c *FargateCompatibilityChecker) CheckDNSOpts(service *types.ServiceConfig) { + if len(service.DNSOpts) > 0 { + c.error("'dns_opt' is not supported") + service.DNSOpts = nil + } +} + +func (c *FargateCompatibilityChecker) CheckExtraHosts(service *types.ServiceConfig) { + if len(service.ExtraHosts) > 0 { + c.error("'extra_hosts' is not supported") + service.ExtraHosts = nil + } +} + +func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) { + for i, v := range service.CapAdd { + if v != "SYS_PTRACE" { + c.error("'cap_add' %s is not supported", v) + l := len(service.CapAdd) + service.CapAdd[i] = service.CapAdd[l-1] + service.CapAdd = service.CapAdd[:l-1] + } + } +} + +func (c *FargateCompatibilityChecker) CheckTmpfs(service *types.ServiceConfig) { + if len(service.Tmpfs) > 0 { + c.error("'tmpfs' is not supported") + service.Tmpfs = nil + } +} + +func (c *FargateCompatibilityChecker) CheckSysctls(service *types.ServiceConfig) { + if len(service.Sysctls) > 0 { + c.error("'sysctls' is not supported") + service.Sysctls = nil + } +} + +func (c *FargateCompatibilityChecker) CheckLabels(service *types.ServiceConfig) { + for k, v := range service.Labels { + if v == "" { + c.error("'labels' with an empty value is not supported") + delete(service.Labels, k) + } + } +} + +var _ CompatibilityChecker = &FargateCompatibilityChecker{} diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 13e90690..7c8724c4 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -10,6 +10,7 @@ import ( ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ecs" + "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/compose-spec/compose-go/types" "github.com/docker/cli/opts" "github.com/docker/ecs-plugin/pkg/compose" @@ -68,7 +69,7 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe ResourceRequirements: nil, StartTimeout: 0, StopTimeout: durationToInt(service.StopGracePeriod), - SystemControls: nil, + SystemControls: toSystemControls(service.Sysctls), Ulimits: toUlimits(service.Ulimits), User: service.User, VolumesFrom: nil, @@ -84,10 +85,32 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe PlacementConstraints: toPlacementConstraints(service.Deploy), ProxyConfiguration: nil, RequiresCompatibilities: []string{ecsapi.LaunchTypeFargate}, - Tags: nil, + Tags: toTags(service.Labels), }, nil } +func toTags(labels types.Labels) []tags.Tag { + t := []tags.Tag{} + for n, v := range labels { + t = append(t, tags.Tag{ + Key: n, + Value: v, + }) + } + return t +} + +func toSystemControls(sysctls types.Mapping) []ecs.TaskDefinition_SystemControl { + sys := []ecs.TaskDefinition_SystemControl{} + for k, v := range sysctls { + sys = append(sys, ecs.TaskDefinition_SystemControl{ + Namespace: k, + Value: v, + }) + } + return sys +} + func toLimits(service types.ServiceConfig) (string, string, error) { // All possible cpu/mem values for Fargate cpuToMem := map[int64][]types.UnitBytes{ diff --git a/ecs/pkg/amazon/validate.go b/ecs/pkg/amazon/validate.go deleted file mode 100644 index 4e7918fc..00000000 --- a/ecs/pkg/amazon/validate.go +++ /dev/null @@ -1,35 +0,0 @@ -package amazon - -import ( - "fmt" - - "github.com/compose-spec/compose-go/types" - "github.com/docker/ecs-plugin/pkg/compose" -) - -// Validate check the compose model do not use unsupported features and inject sane defaults for ECS deployment -func Validate(project *compose.Project) error { - if len(project.Networks) == 0 { - // Compose application model implies a default network if none is explicitly set. - // FIXME move this to compose-go - project.Networks["default"] = types.NetworkConfig{ - Name: "default", - } - } - - for i, service := range project.Services { - if len(service.Networks) == 0 { - // Service without explicit network attachment are implicitly exposed on default network - // FIXME move this to compose-go - service.Networks = map[string]*types.ServiceNetworkConfig{"default": nil} - project.Services[i] = service - } - - if service.NetworkMode != "" && service.NetworkMode != "awsvpc" { - return fmt.Errorf("ECS do not support NetworkMode %q", service.NetworkMode) - } - } - - // Here we can check for incompatible attributes, inject sane defaults, etc - return nil -} diff --git a/ecs/pkg/compose/normalize.go b/ecs/pkg/compose/normalize.go new file mode 100644 index 00000000..0061a1f1 --- /dev/null +++ b/ecs/pkg/compose/normalize.go @@ -0,0 +1,53 @@ +package compose + +import ( + "fmt" + + "github.com/compose-spec/compose-go/types" + "github.com/sirupsen/logrus" +) + +// Normalize a compose-go model to move deprecated attributes to canonical position, and introduce implicit defaults +// FIXME move this to compose-go +func Normalize(model *types.Config) error { + if len(model.Networks) == 0 { + // Compose application model implies a default network if none is explicitly set. + model.Networks["default"] = types.NetworkConfig{ + Name: "default", + } + } + + for i, s := range model.Services { + if len(s.Networks) == 0 { + // Service without explicit network attachment are implicitly exposed on default network + s.Networks = map[string]*types.ServiceNetworkConfig{"default": nil} + } + + if s.LogDriver != "" { + logrus.Warn("`log_driver` is deprecated. Use the `logging` attribute") + if s.Logging == nil { + s.Logging = &types.LoggingConfig{} + } + if s.Logging.Driver == "" { + s.Logging.Driver = s.LogDriver + } else { + return fmt.Errorf("can't use both 'log_driver' (deprecated) and 'logging.driver'") + } + } + if len(s.LogOpt) != 0 { + logrus.Warn("`log_opts` is deprecated. Use the `logging` attribute") + if s.Logging == nil { + s.Logging = &types.LoggingConfig{} + } + for k, v := range s.LogOpt { + if _, ok := s.Logging.Options[k]; !ok { + s.Logging.Options[k] = v + } else { + return fmt.Errorf("can't use both 'log_opt' (deprecated) and 'logging.options'") + } + } + } + model.Services[i] = s + } + return nil +} diff --git a/ecs/pkg/compose/project.go b/ecs/pkg/compose/project.go index 5c244b2c..a90bba07 100644 --- a/ecs/pkg/compose/project.go +++ b/ecs/pkg/compose/project.go @@ -25,6 +25,11 @@ func NewProject(config types.ConfigDetails, name string) (*Project, error) { return nil, err } + err = Normalize(model) + if err != nil { + return nil, err + } + p := Project{ Config: *model, projectDir: config.WorkingDir, From e9fe3b28643f595c9d9fe027d8947f12d164e1cb Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Tue, 19 May 2020 11:49:58 +0200 Subject: [PATCH 078/205] Add e2e test deploying a compose application to an ECS cluster Signed-off-by: Guillaume Lours Signed-off-by: Nicolas De Loof --- ecs/Makefile | 5 +- ecs/pkg/amazon/api.go | 5 + ecs/pkg/amazon/mock/api.go | 101 ++++++++++++------ ecs/tests/command_test.go | 2 +- ecs/tests/e2e_deploy_services_test.go | 65 +++++++++++ ecs/tests/main_test.go | 9 +- ecs/tests/plugin_test.go | 2 +- ecs/tests/setup_command_test.go | 2 +- .../testdata/input/simple-single-service.yaml | 6 ++ 9 files changed, 157 insertions(+), 40 deletions(-) create mode 100644 ecs/tests/e2e_deploy_services_test.go create mode 100644 ecs/tests/testdata/input/simple-single-service.yaml diff --git a/ecs/Makefile b/ecs/Makefile index 126d5bc5..1033acd2 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -7,10 +7,13 @@ build: test: build ## Run tests go test ./... -v +e2e: build ## Run tests + go test ./... -v -tags=e2e + dev: build ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" lint: ## Verify Go files golangci-lint run --config ./golangci.yaml ./... -.PHONY: clean build test dev lint +.PHONY: clean build test dev lint e2e diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index 4fa6ddc4..d32e7b7f 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -1,5 +1,7 @@ package amazon +import "context" + //go:generate mockgen -destination=./mock/api.go -package=mock . API type API interface { @@ -7,4 +9,7 @@ type API interface { upAPI logsAPI secretsAPI + GetTasks(ctx context.Context, cluster string, name string) ([]string, error) + GetNetworkInterfaces(ctx context.Context, cluster string, arns ...string) ([]string, error) + GetPublicIPs(ctx context.Context, interfaces ...string) ([]string, error) } diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/mock/api.go index c592162b..d834ec62 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/mock/api.go @@ -6,12 +6,11 @@ package mock import ( context "context" - reflect "reflect" - cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" docker "github.com/docker/ecs-plugin/pkg/docker" gomock "github.com/golang/mock/gomock" + reflect "reflect" ) // MockAPI is a mock of API interface @@ -52,21 +51,6 @@ func (mr *MockAPIMockRecorder) ClusterExists(arg0, arg1 interface{}) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), arg0, arg1) } -// CreateCluster mocks base method -func (m *MockAPI) CreateCluster(arg0 context.Context, arg1 string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateCluster", arg0, arg1) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateCluster indicates an expected call of CreateCluster -func (mr *MockAPIMockRecorder) CreateCluster(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateCluster", reflect.TypeOf((*MockAPI)(nil).CreateCluster), arg0, arg1) -} - // CreateSecret mocks base method func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 docker.Secret) (string, error) { m.ctrl.T.Helper() @@ -168,6 +152,60 @@ func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC), arg0) } +// GetLogs mocks base method +func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLogs", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// GetLogs indicates an expected call of GetLogs +func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1) +} + +// GetNetworkInterfaces mocks base method +func (m *MockAPI) GetNetworkInterfaces(arg0 context.Context, arg1 string, arg2 ...string) ([]string, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetNetworkInterfaces", varargs...) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNetworkInterfaces indicates an expected call of GetNetworkInterfaces +func (mr *MockAPIMockRecorder) GetNetworkInterfaces(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetworkInterfaces", reflect.TypeOf((*MockAPI)(nil).GetNetworkInterfaces), varargs...) +} + +// GetPublicIPs mocks base method +func (m *MockAPI) GetPublicIPs(arg0 context.Context, arg1 ...string) ([]string, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetPublicIPs", varargs...) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPublicIPs indicates an expected call of GetPublicIPs +func (mr *MockAPIMockRecorder) GetPublicIPs(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPublicIPs", reflect.TypeOf((*MockAPI)(nil).GetPublicIPs), varargs...) +} + // GetStackID mocks base method func (m *MockAPI) GetStackID(arg0 context.Context, arg1 string) (string, error) { m.ctrl.T.Helper() @@ -198,6 +236,21 @@ func (mr *MockAPIMockRecorder) GetSubNets(arg0, arg1 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), arg0, arg1) } +// GetTasks mocks base method +func (m *MockAPI) GetTasks(arg0 context.Context, arg1, arg2 string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTasks", arg0, arg1, arg2) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTasks indicates an expected call of GetTasks +func (mr *MockAPIMockRecorder) GetTasks(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTasks", reflect.TypeOf((*MockAPI)(nil).GetTasks), arg0, arg1, arg2) +} + // InspectSecret mocks base method func (m *MockAPI) InspectSecret(arg0 context.Context, arg1 string) (docker.Secret, error) { m.ctrl.T.Helper() @@ -271,17 +324,3 @@ func (mr *MockAPIMockRecorder) WaitStackComplete(arg0, arg1, arg2 interface{}) * mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitStackComplete", reflect.TypeOf((*MockAPI)(nil).WaitStackComplete), arg0, arg1, arg2) } - -// GetLogs mocks base method -func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLogs", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// GetLogs mocks base method -func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1) -} diff --git a/ecs/tests/command_test.go b/ecs/tests/command_test.go index a027fbae..3ca7433d 100644 --- a/ecs/tests/command_test.go +++ b/ecs/tests/command_test.go @@ -7,7 +7,7 @@ import ( ) func TestExitErrorCode(t *testing.T) { - cmd, cleanup := dockerCli.createTestCmd() + cmd, cleanup, _ := dockerCli.createTestCmd() defer cleanup() cmd.Command = dockerCli.Command("ecs", "unknown_command") diff --git a/ecs/tests/e2e_deploy_services_test.go b/ecs/tests/e2e_deploy_services_test.go new file mode 100644 index 00000000..80d77f31 --- /dev/null +++ b/ecs/tests/e2e_deploy_services_test.go @@ -0,0 +1,65 @@ +// +build e2e + +package tests + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/docker/ecs-plugin/pkg/amazon" + "github.com/docker/ecs-plugin/pkg/docker" + "gotest.tools/assert" + "gotest.tools/v3/fs" + "gotest.tools/v3/golden" + "gotest.tools/v3/icmd" +) + +const ( + composeFileName = "compose.yaml" +) + +func TestE2eDeployServices(t *testing.T) { + cmd, cleanup, awsContext := dockerCli.createTestCmd() + defer cleanup() + + composeUpSimpleService(t, cmd, awsContext) +} + +func composeUpSimpleService(t *testing.T, cmd icmd.Cmd, awsContext docker.AwsContext) { + bgContext := context.Background() + composeYAML := golden.Get(t, "input/simple-single-service.yaml") + tmpDir := fs.NewDir(t, t.Name(), + fs.WithFile(composeFileName, "", fs.WithBytes(composeYAML)), + ) + // We can't use the file added in the tmp directory because it will drop if an assertion fails + defer composeDown(t, cmd, golden.Path("input/simple-single-service.yaml")) + defer tmpDir.Remove() + + cmd.Command = dockerCli.Command("ecs", "compose", "--file="+tmpDir.Join(composeFileName), "--project-name", t.Name(), "up") + icmd.RunCmd(cmd).Assert(t, icmd.Success) + + session, err := session.NewSessionWithOptions(session.Options{ + Profile: awsContext.Profile, + Config: aws.Config{ + Region: aws.String(awsContext.Region), + }, + }) + assert.NilError(t, err) + sdk := amazon.NewAPI(session) + arns, err := sdk.GetTasks(bgContext, t.Name(), "simple") + assert.NilError(t, err) + networkInterfaces, err := sdk.GetNetworkInterfaces(bgContext, t.Name(), arns...) + publicIps, err := sdk.GetPublicIPs(context.Background(), networkInterfaces...) + assert.NilError(t, err) + for _, ip := range publicIps { + icmd.RunCommand("curl", "-I", "http://"+ip).Assert(t, icmd.Success) + } + +} + +func composeDown(t *testing.T, cmd icmd.Cmd, composeFile string) { + cmd.Command = dockerCli.Command("ecs", "compose", "--file="+composeFile, "--project-name", t.Name(), "down") + icmd.RunCmd(cmd).Assert(t, icmd.Success) +} diff --git a/ecs/tests/main_test.go b/ecs/tests/main_test.go index a093c6c9..b8a75814 100644 --- a/ecs/tests/main_test.go +++ b/ecs/tests/main_test.go @@ -29,7 +29,7 @@ type dockerCliCommand struct { type ConfigFileOperator func(configFile *dockerConfigFile.ConfigFile) -func (d dockerCliCommand) createTestCmd(ops ...ConfigFileOperator) (icmd.Cmd, func()) { +func (d dockerCliCommand) createTestCmd(ops ...ConfigFileOperator) (icmd.Cmd, func(), docker.AwsContext) { configDir, err := ioutil.TempDir("", "config") if err != nil { panic(err) @@ -55,9 +55,8 @@ func (d dockerCliCommand) createTestCmd(ops ...ConfigFileOperator) (icmd.Cmd, fu } awsContext := docker.AwsContext{ - Profile: "TestProfile", - Cluster: "TestCluster", - Region: "TestRegion", + Profile: "sandbox.devtools.developer", + Region: "eu-west-3", } testStore, err := docker.NewContextWithStore(testContextName, &awsContext, filepath.Join(configDir, "contexts")) if err != nil { @@ -71,7 +70,7 @@ func (d dockerCliCommand) createTestCmd(ops ...ConfigFileOperator) (icmd.Cmd, fu env := append(os.Environ(), "DOCKER_CONFIG="+configDir, "DOCKER_CLI_EXPERIMENTAL=enabled") // TODO: Remove this once docker ecs plugin is no more experimental - return icmd.Cmd{Env: env}, cleanup + return icmd.Cmd{Env: env}, cleanup, awsContext } func (d dockerCliCommand) Command(args ...string) []string { diff --git a/ecs/tests/plugin_test.go b/ecs/tests/plugin_test.go index f519c252..48d7e2f5 100644 --- a/ecs/tests/plugin_test.go +++ b/ecs/tests/plugin_test.go @@ -10,7 +10,7 @@ import ( ) func TestInvokePluginFromCLI(t *testing.T) { - cmd, cleanup := dockerCli.createTestCmd() + cmd, cleanup, _ := dockerCli.createTestCmd() defer cleanup() // docker --help should list app as a top command cmd.Command = dockerCli.Command("--help") diff --git a/ecs/tests/setup_command_test.go b/ecs/tests/setup_command_test.go index a0cc9e46..d36393f4 100644 --- a/ecs/tests/setup_command_test.go +++ b/ecs/tests/setup_command_test.go @@ -10,7 +10,7 @@ import ( ) func TestDefaultAwsContextName(t *testing.T) { - cmd, cleanup := dockerCli.createTestCmd() + cmd, cleanup, _ := dockerCli.createTestCmd() defer cleanup() cmd.Command = dockerCli.Command("ecs", "setup", "--cluster", "clusterName", "--profile", "profileName", diff --git a/ecs/tests/testdata/input/simple-single-service.yaml b/ecs/tests/testdata/input/simple-single-service.yaml new file mode 100644 index 00000000..95f30b1d --- /dev/null +++ b/ecs/tests/testdata/input/simple-single-service.yaml @@ -0,0 +1,6 @@ +version: "3" +services: + simple: + image: nginx + ports: + - 80:80 \ No newline at end of file From 3283bceac6cae74a02561438d83f8c934d074a2d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 25 May 2020 16:49:58 +0200 Subject: [PATCH 079/205] Support pull from ECR close #58 Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 1 + ecs/pkg/amazon/iam.go | 1 + .../testdata/simple/simple-cloudformation-conversion.golden | 3 ++- .../simple-cloudformation-with-overrides-conversion.golden | 3 ++- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 2eb6f1df..eec6b854 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -120,6 +120,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Policies: rolePolicies, ManagedPolicyArns: []string{ ECSTaskExecutionPolicy, + ECRReadOnlyPolicy, }, } template.Resources[taskDefinition] = definition diff --git a/ecs/pkg/amazon/iam.go b/ecs/pkg/amazon/iam.go index 66357730..affcaaae 100644 --- a/ecs/pkg/amazon/iam.go +++ b/ecs/pkg/amazon/iam.go @@ -2,6 +2,7 @@ package amazon const ( ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + ECRReadOnlyPolicy = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" ActionGetSecretValue = "secretsmanager:GetSecretValue" ActionGetParameters = "ssm:GetParameters" diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 0910aaf3..0050bca0 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -217,7 +217,8 @@ "Version": "2012-10-17" }, "ManagedPolicyArns": [ - "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" ] }, "Type": "AWS::IAM::Role" diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 7c413438..328d627a 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -217,7 +217,8 @@ "Version": "2012-10-17" }, "ManagedPolicyArns": [ - "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" ] }, "Type": "AWS::IAM::Role" From a798c95963d67883e3f0aa0c070dae24a910fac5 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 27 May 2020 14:34:17 +0200 Subject: [PATCH 080/205] Register services with a known port with SRV record see https://github.com/docker/docker_aws/issues/15#issuecomment-634357859 Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 33 ++++++++++++------- .../simple-cloudformation-conversion.golden | 2 +- ...formation-with-overrides-conversion.golden | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index eec6b854..48960592 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -131,18 +131,31 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", service.Name) + records := []cloudmap.Service_DnsRecord{ + { + TTL: 60, + Type: cloudmapapi.RecordTypeA, + }, + } + serviceRegistry := ecs.Service_ServiceRegistry{ + RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), + } + + if len(service.Ports) > 0 { + records = append(records, cloudmap.Service_DnsRecord{ + TTL: 60, + Type: cloudmapapi.RecordTypeSrv, + }) + serviceRegistry.Port = int(service.Ports[0].Target) + } + template.Resources[serviceRegistration] = &cloudmap.Service{ Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), HealthCheckConfig: healthCheck, Name: service.Name, NamespaceId: cloudformation.Ref("CloudMap"), DnsConfig: &cloudmap.Service_DnsConfig{ - DnsRecords: []cloudmap.Service_DnsRecord{ - { - TTL: 300, - Type: cloudmapapi.RecordTypeA, - }, - }, + DnsRecords: records, RoutingPolicy: cloudmapapi.RoutingPolicyMultivalue, }, } @@ -169,12 +182,8 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err }, SchedulingStrategy: ecsapi.SchedulingStrategyReplica, ServiceName: service.Name, - ServiceRegistries: []ecs.Service_ServiceRegistry{ - { - RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), - }, - }, - TaskDefinition: cloudformation.Ref(taskDefinition), + ServiceRegistries: []ecs.Service_ServiceRegistry{serviceRegistry}, + TaskDefinition: cloudformation.Ref(taskDefinition), } } return template, nil diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 0050bca0..868dcdd0 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -135,7 +135,7 @@ "DnsConfig": { "DnsRecords": [ { - "TTL": 300, + "TTL": 60, "Type": "A" } ], diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 328d627a..fe41a67f 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -135,7 +135,7 @@ "DnsConfig": { "DnsRecords": [ { - "TTL": 300, + "TTL": 60, "Type": "A" } ], From 3bc5fc129ebc713a0aaafe03e5d87c56a8a404c9 Mon Sep 17 00:00:00 2001 From: Chad Metcalf Date: Tue, 26 May 2020 12:01:43 -0700 Subject: [PATCH 081/205] Create the plugin directory if it doesn't exist. Signed-off-by: Nicolas De Loof --- ecs/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs/Makefile b/ecs/Makefile index 1033acd2..0fd6b65a 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -11,6 +11,7 @@ e2e: build ## Run tests go test ./... -v -tags=e2e dev: build + @mkdir -p ~/.docker/cli-plugins/ ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" lint: ## Verify Go files From 01e2b0c989ff6c3f798e162cd2f13f0affa8f469 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 27 May 2020 16:50:04 +0200 Subject: [PATCH 082/205] Present service logs with colored service prefix This reproduce docker-compose behaviour to report logs with prefix also moves log formating out from sdk.go Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/api.go | 2 +- ecs/pkg/amazon/{mock/api.go => api_mock.go} | 12 ++-- ecs/pkg/amazon/down_test.go | 5 +- ecs/pkg/amazon/logs.go | 54 +++++++++++++++++- ecs/pkg/amazon/sdk.go | 6 +- ecs/pkg/console/colors.go | 62 +++++++++++++++++++++ 6 files changed, 127 insertions(+), 14 deletions(-) rename ecs/pkg/amazon/{mock/api.go => api_mock.go} (97%) create mode 100644 ecs/pkg/console/colors.go diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index d32e7b7f..fecc5d80 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -2,7 +2,7 @@ package amazon import "context" -//go:generate mockgen -destination=./mock/api.go -package=mock . API +//go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon" -package=amazon . API type API interface { downAPI diff --git a/ecs/pkg/amazon/mock/api.go b/ecs/pkg/amazon/api_mock.go similarity index 97% rename from ecs/pkg/amazon/mock/api.go rename to ecs/pkg/amazon/api_mock.go index d834ec62..06ef4fc0 100644 --- a/ecs/pkg/amazon/mock/api.go +++ b/ecs/pkg/amazon/api_mock.go @@ -1,8 +1,8 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/docker/ecs-plugin/pkg/amazon (interfaces: API) -// Package mock is a generated GoMock package. -package mock +// Package amazon is a generated GoMock package. +package amazon import ( context "context" @@ -153,17 +153,17 @@ func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call { } // GetLogs mocks base method -func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string) error { +func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 LogConsumer) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLogs", arg0, arg1) + ret := m.ctrl.Call(m, "GetLogs", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // GetLogs indicates an expected call of GetLogs -func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1, arg2) } // GetNetworkInterfaces mocks base method diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/down_test.go index bf98a6b9..642faf75 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/down_test.go @@ -4,14 +4,13 @@ import ( "context" "testing" - "github.com/docker/ecs-plugin/pkg/amazon/mock" "github.com/golang/mock/gomock" ) func TestDownDontDeleteCluster(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - m := mock.NewMockAPI(ctrl) + m := NewMockAPI(ctrl) c := &client{ Cluster: "test_cluster", Region: "region", @@ -30,7 +29,7 @@ func TestDownDontDeleteCluster(t *testing.T) { func TestDownDeleteCluster(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - m := mock.NewMockAPI(ctrl) + m := NewMockAPI(ctrl) c := &client{ Cluster: "test_cluster", Region: "region", diff --git a/ecs/pkg/amazon/logs.go b/ecs/pkg/amazon/logs.go index b515e25d..683ecc26 100644 --- a/ecs/pkg/amazon/logs.go +++ b/ecs/pkg/amazon/logs.go @@ -2,12 +2,62 @@ package amazon import ( "context" + "fmt" + "os" + "os/signal" + "strconv" + "strings" + + "github.com/docker/ecs-plugin/pkg/console" ) func (c *client) ComposeLogs(ctx context.Context, projectName string) error { - return c.api.GetLogs(ctx, projectName) + err := c.api.GetLogs(ctx, projectName, &logConsumer{ + colors: map[string]console.ColorFunc{}, + width: 0, + }) + if err != nil { + return err + } + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt) + <-signalChan + return nil +} + +type logConsumer struct { + colors map[string]console.ColorFunc + width int +} + +func (l *logConsumer) Log(service, container, message string) { + cf, ok := l.colors[service] + if !ok { + cf = <-console.Rainbow + l.colors[service] = cf + l.computeWidth() + } + prefix := fmt.Sprintf("%-"+strconv.Itoa(l.width)+"s |", service) + for _, line := range strings.Split(message, "\n") { + fmt.Printf("%s %s\n", cf(prefix), line) + } +} + +func (l *logConsumer) computeWidth() { + width := 0 + for n := range l.colors { + if len(n) > width { + width = len(n) + } + } + l.width = width + 3 +} + +type LogConsumer interface { + Log(service, container, message string) } type logsAPI interface { - GetLogs(ctx context.Context, name string) error + GetLogs(ctx context.Context, name string, consumer LogConsumer) error } diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 2df26efd..42b00c8e 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -3,6 +3,7 @@ package amazon import ( "context" "fmt" + "strings" "time" "github.com/aws/aws-sdk-go/aws" @@ -309,7 +310,7 @@ func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { return err } -func (s sdk) GetLogs(ctx context.Context, name string) error { +func (s sdk) GetLogs(ctx context.Context, name string, consumer LogConsumer) error { logGroup := fmt.Sprintf("/docker-compose/%s", name) var startTime int64 for { @@ -331,7 +332,8 @@ func (s sdk) GetLogs(ctx context.Context, name string) error { } for _, event := range events.Events { - fmt.Println(*event.Message) + p := strings.Split(*event.LogStreamName, "/") + consumer.Log(p[1], p[2], *event.Message) startTime = *event.IngestionTime } } diff --git a/ecs/pkg/console/colors.go b/ecs/pkg/console/colors.go new file mode 100644 index 00000000..517afb67 --- /dev/null +++ b/ecs/pkg/console/colors.go @@ -0,0 +1,62 @@ +package console + +import ( + "strconv" +) + +var NAMES = []string{ + "grey", + "red", + "green", + "yellow", + "blue", + "magenta", + "cyan", + "white", +} + +var COLORS map[string]ColorFunc + +// ColorFunc use ANSI codes to render colored text on console +type ColorFunc func(s string) string + +var Monochrome = func(s string) string { + return s +} + +func makeColorFunc(code string) ColorFunc { + return func(s string) string { + return ansiColor(code, s) + } +} + +var Rainbow = make(chan ColorFunc) + +func init() { + COLORS = map[string]ColorFunc{} + for i, name := range NAMES { + COLORS[name] = makeColorFunc(strconv.Itoa(30 + i)) + COLORS["intense_"+name] = makeColorFunc(strconv.Itoa(30+i) + ";1") + } + + go func() { + i := 0 + rainbow := []ColorFunc{ + COLORS["cyan"], + COLORS["yellow"], + COLORS["green"], + COLORS["magenta"], + COLORS["blue"], + COLORS["intense_cyan"], + COLORS["intense_yellow"], + COLORS["intense_green"], + COLORS["intense_magenta"], + COLORS["intense_blue"], + } + + for { + Rainbow <- rainbow[i] + i = (i + 1) % len(rainbow) + } + }() +} From 257f8296794709f33a32ac01c8359f0721c16fa4 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 27 May 2020 17:00:25 +0200 Subject: [PATCH 083/205] Create service with project and service tags Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/client.go | 1 + ecs/pkg/amazon/cloudformation.go | 12 +++++++++++- .../simple/simple-cloudformation-conversion.golden | 10 ++++++++++ ...e-cloudformation-with-overrides-conversion.golden | 10 ++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/client.go index 53b40af6..839c5118 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/client.go @@ -9,6 +9,7 @@ import ( const ( ProjectTag = "com.docker.compose.project" NetworkTag = "com.docker.compose.network" + ServiceTag = "com.docker.compose.service" ) func NewClient(profile string, cluster string, region string) (compose.API, error) { diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 48960592..e838f14a 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -183,7 +183,17 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err SchedulingStrategy: ecsapi.SchedulingStrategyReplica, ServiceName: service.Name, ServiceRegistries: []ecs.Service_ServiceRegistry{serviceRegistry}, - TaskDefinition: cloudformation.Ref(taskDefinition), + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + { + Key: ServiceTag, + Value: service.Name, + }, + }, + TaskDefinition: cloudformation.Ref(taskDefinition), } } return template, nil diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 868dcdd0..bbe34758 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -123,6 +123,16 @@ } } ], + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleConvert" + }, + { + "Key": "com.docker.compose.service", + "Value": "simple" + } + ], "TaskDefinition": { "Ref": "simpleTaskDefinition" } diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index fe41a67f..d9e57b00 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -123,6 +123,16 @@ } } ], + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleWithOverrides" + }, + { + "Key": "com.docker.compose.service", + "Value": "simple" + } + ], "TaskDefinition": { "Ref": "simpleTaskDefinition" } From 564c369c3e617c34a30b2bff7c0b7e0a8e2d49c1 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 27 May 2020 16:58:58 +0200 Subject: [PATCH 084/205] Compute resource names to avoid unsupported characters Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 17 ++++-- ecs/pkg/amazon/convert.go | 2 +- .../simple-cloudformation-conversion.golden | 56 +++++++++---------- ...formation-with-overrides-conversion.golden | 56 +++++++++---------- 4 files changed, 68 insertions(+), 63 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index e838f14a..9df24750 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -2,6 +2,7 @@ package amazon import ( "fmt" + "regexp" "strings" "github.com/sirupsen/logrus" @@ -99,7 +100,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return nil, err } - taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", service.Name) + taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", normalizeResourceName(service.Name)) policy, err := c.getPolicy(definition) if err != nil { return nil, err @@ -114,7 +115,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } definition.ExecutionRoleArn = cloudformation.Ref(taskExecutionRole) - taskDefinition := fmt.Sprintf("%sTaskDefinition", service.Name) + taskDefinition := fmt.Sprintf("%sTaskDefinition", normalizeResourceName(service.Name)) template.Resources[taskExecutionRole] = &iam.Role{ AssumeRolePolicyDocument: assumeRolePolicyDocument, Policies: rolePolicies, @@ -130,7 +131,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err // FIXME ECS only support HTTP(s) health checks, while Docker only support CMD } - serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", service.Name) + serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", normalizeResourceName(service.Name)) records := []cloudmap.Service_DnsRecord{ { TTL: 60, @@ -166,7 +167,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err serviceSecurityGroups = append(serviceSecurityGroups, cloudformation.Ref(logicalName)) } - template.Resources[fmt.Sprintf("%sService", service.Name)] = &ecs.Service{ + template.Resources[fmt.Sprintf("%sService", normalizeResourceName(service.Name))] = &ecs.Service{ Cluster: cluster, DesiredCount: 1, LaunchType: ecsapi.LaunchTypeFargate, @@ -193,7 +194,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Value: service.Name, }, }, - TaskDefinition: cloudformation.Ref(taskDefinition), + TaskDefinition: cloudformation.Ref(normalizeResourceName(taskDefinition)), } } return template, nil @@ -236,7 +237,11 @@ func convertNetwork(project *compose.Project, net string, vpc string) (string, c } func networkResourceName(project *compose.Project, network string) string { - return fmt.Sprintf("%s%sNetwork", project.Name, strings.Title(network)) + return fmt.Sprintf("%s%sNetwork", normalizeResourceName(project.Name), normalizeResourceName(network)) +} + +func normalizeResourceName(s string) string { + return strings.Title(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(s, "")) } func (c client) getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 7c8724c4..93d546a6 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -57,7 +57,7 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe Options: map[string]string{ "awslogs-region": cloudformation.Ref("AWS::Region"), "awslogs-group": cloudformation.Ref("LogGroup"), - "awslogs-stream-prefix": service.Name, + "awslogs-stream-prefix": project.Name, }, }, Name: service.Name, diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index bbe34758..d78914e6 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -58,27 +58,7 @@ }, "Type": "AWS::Logs::LogGroup" }, - "TestSimpleConvertDefaultNetwork": { - "Properties": { - "GroupDescription": "TestSimpleConvert default Security Group", - "GroupName": "TestSimpleConvertDefaultNetwork", - "Tags": [ - { - "Key": "com.docker.compose.project", - "Value": "TestSimpleConvert" - }, - { - "Key": "com.docker.compose.network", - "Value": "default" - } - ], - "VpcId": { - "Ref": "ParameterVPCId" - } - }, - "Type": "AWS::EC2::SecurityGroup" - }, - "simpleService": { + "SimpleService": { "Properties": { "Cluster": { "Fn::If": [ @@ -117,7 +97,7 @@ { "RegistryArn": { "Fn::GetAtt": [ - "simpleServiceDiscoveryEntry", + "SimpleServiceDiscoveryEntry", "Arn" ] } @@ -134,12 +114,12 @@ } ], "TaskDefinition": { - "Ref": "simpleTaskDefinition" + "Ref": "SimpleTaskDefinition" } }, "Type": "AWS::ECS::Service" }, - "simpleServiceDiscoveryEntry": { + "SimpleServiceDiscoveryEntry": { "Properties": { "Description": "\"simple\" service discovery entry in Cloud Map", "DnsConfig": { @@ -158,7 +138,7 @@ }, "Type": "AWS::ServiceDiscovery::Service" }, - "simpleTaskDefinition": { + "SimpleTaskDefinition": { "Properties": { "ContainerDefinitions": [ { @@ -191,7 +171,7 @@ "awslogs-region": { "Ref": "AWS::Region" }, - "awslogs-stream-prefix": "simple" + "awslogs-stream-prefix": "TestSimpleConvert" } }, "Name": "simple" @@ -199,7 +179,7 @@ ], "Cpu": "256", "ExecutionRoleArn": { - "Ref": "simpleTaskExecutionRole" + "Ref": "SimpleTaskExecutionRole" }, "Family": "TestSimpleConvert-simple", "Memory": "512", @@ -210,7 +190,7 @@ }, "Type": "AWS::ECS::TaskDefinition" }, - "simpleTaskExecutionRole": { + "SimpleTaskExecutionRole": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -232,6 +212,26 @@ ] }, "Type": "AWS::IAM::Role" + }, + "TestSimpleConvertDefaultNetwork": { + "Properties": { + "GroupDescription": "TestSimpleConvert default Security Group", + "GroupName": "TestSimpleConvertDefaultNetwork", + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleConvert" + }, + { + "Key": "com.docker.compose.network", + "Value": "default" + } + ], + "VpcId": { + "Ref": "ParameterVPCId" + } + }, + "Type": "AWS::EC2::SecurityGroup" } } } diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index d9e57b00..1eaeed17 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -58,27 +58,7 @@ }, "Type": "AWS::Logs::LogGroup" }, - "TestSimpleWithOverridesDefaultNetwork": { - "Properties": { - "GroupDescription": "TestSimpleWithOverrides default Security Group", - "GroupName": "TestSimpleWithOverridesDefaultNetwork", - "Tags": [ - { - "Key": "com.docker.compose.project", - "Value": "TestSimpleWithOverrides" - }, - { - "Key": "com.docker.compose.network", - "Value": "default" - } - ], - "VpcId": { - "Ref": "ParameterVPCId" - } - }, - "Type": "AWS::EC2::SecurityGroup" - }, - "simpleService": { + "SimpleService": { "Properties": { "Cluster": { "Fn::If": [ @@ -117,7 +97,7 @@ { "RegistryArn": { "Fn::GetAtt": [ - "simpleServiceDiscoveryEntry", + "SimpleServiceDiscoveryEntry", "Arn" ] } @@ -134,12 +114,12 @@ } ], "TaskDefinition": { - "Ref": "simpleTaskDefinition" + "Ref": "SimpleTaskDefinition" } }, "Type": "AWS::ECS::Service" }, - "simpleServiceDiscoveryEntry": { + "SimpleServiceDiscoveryEntry": { "Properties": { "Description": "\"simple\" service discovery entry in Cloud Map", "DnsConfig": { @@ -158,7 +138,7 @@ }, "Type": "AWS::ServiceDiscovery::Service" }, - "simpleTaskDefinition": { + "SimpleTaskDefinition": { "Properties": { "ContainerDefinitions": [ { @@ -191,7 +171,7 @@ "awslogs-region": { "Ref": "AWS::Region" }, - "awslogs-stream-prefix": "simple" + "awslogs-stream-prefix": "TestSimpleWithOverrides" } }, "Name": "simple" @@ -199,7 +179,7 @@ ], "Cpu": "256", "ExecutionRoleArn": { - "Ref": "simpleTaskExecutionRole" + "Ref": "SimpleTaskExecutionRole" }, "Family": "TestSimpleWithOverrides-simple", "Memory": "512", @@ -210,7 +190,7 @@ }, "Type": "AWS::ECS::TaskDefinition" }, - "simpleTaskExecutionRole": { + "SimpleTaskExecutionRole": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -232,6 +212,26 @@ ] }, "Type": "AWS::IAM::Role" + }, + "TestSimpleWithOverridesDefaultNetwork": { + "Properties": { + "GroupDescription": "TestSimpleWithOverrides default Security Group", + "GroupName": "TestSimpleWithOverridesDefaultNetwork", + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleWithOverrides" + }, + { + "Key": "com.docker.compose.network", + "Value": "default" + } + ], + "VpcId": { + "Ref": "ParameterVPCId" + } + }, + "Type": "AWS::EC2::SecurityGroup" } } } From da299f59e2a941dc7e2c903111fc114407eb01b8 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 27 May 2020 14:38:50 +0200 Subject: [PATCH 085/205] introduce 'ps' command Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 21 ++++++++++++++ ecs/pkg/amazon/api.go | 8 ++---- ecs/pkg/amazon/list.go | 55 +++++++++++++++++++++++++++++++++++++ ecs/pkg/amazon/sdk.go | 4 ++- ecs/pkg/compose/api.go | 1 + 5 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 ecs/pkg/amazon/list.go diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index f7a07226..0b1e70e6 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -23,6 +23,7 @@ func ComposeCommand(dockerCli command.Cli) *cobra.Command { UpCommand(dockerCli, opts), DownCommand(dockerCli, opts), LogsCommand(dockerCli, opts), + PsCommand(dockerCli, opts), ) return cmd } @@ -87,6 +88,26 @@ func UpCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobr return cmd } +func PsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { + opts := upOptions{} + cmd := &cobra.Command{ + Use: "ps", + RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + clusteropts, err := docker.GetAwsContext(dockerCli) + if err != nil { + return err + } + client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + if err != nil { + return err + } + return client.ComposePs(context.Background(), project) + }), + } + cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") + return cmd +} + type downOptions struct { DeleteCluster bool } diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index fecc5d80..9fd599a1 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -1,15 +1,11 @@ package amazon -import "context" - -//go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon" -package=amazon . API +//go:generate mockgen -destination=./mock/api.go -package=mock . API type API interface { downAPI upAPI logsAPI secretsAPI - GetTasks(ctx context.Context, cluster string, name string) ([]string, error) - GetNetworkInterfaces(ctx context.Context, cluster string, arns ...string) ([]string, error) - GetPublicIPs(ctx context.Context, interfaces ...string) ([]string, error) + psAPI } diff --git a/ecs/pkg/amazon/list.go b/ecs/pkg/amazon/list.go new file mode 100644 index 00000000..d401e0d6 --- /dev/null +++ b/ecs/pkg/amazon/list.go @@ -0,0 +1,55 @@ +package amazon + +import ( + "context" + "fmt" + "os" + "strings" + "text/tabwriter" + + "github.com/docker/ecs-plugin/pkg/compose" +) + +func (c *client) ComposePs(ctx context.Context, project *compose.Project) error { + cluster := c.Cluster + if cluster == "" { + cluster = project.Name + } + w := tabwriter.NewWriter(os.Stdout, 20, 2, 3, ' ', 0) + fmt.Fprintf(w, "Name\tState\tPorts\n") + for _, s := range project.Services { + tasks, err := c.api.GetTasks(ctx, cluster, s.Name) + if err != nil { + return err + } + if len(tasks) == 0 { + continue + } + // TODO get more data from DescribeTask, including tasks status + networkInterfaces, err := c.api.GetNetworkInterfaces(ctx, cluster, tasks...) + if err != nil { + return err + } + if len(networkInterfaces) == 0 { + fmt.Fprintf(w, "%s\t%s\t\n", s.Name, "Provisioning") + continue + } + publicIps, err := c.api.GetPublicIPs(ctx, networkInterfaces...) + if err != nil { + return err + } + ports := []string{} + for _, p := range s.Ports { + ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", strings.Join(publicIps, ","), p.Published, p.Target, p.Protocol)) + } + fmt.Fprintf(w, "%s\t%s\t%s\n", s.Name, "Up", strings.Join(ports, ", ")) + } + w.Flush() + return nil +} + +type psAPI interface { + GetTasks(ctx context.Context, cluster string, name string) ([]string, error) + GetNetworkInterfaces(ctx context.Context, cluster string, arns ...string) ([]string, error) + GetPublicIPs(ctx context.Context, interfaces ...string) ([]string, error) +} diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 42b00c8e..3f9b9e6b 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -388,7 +388,9 @@ func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) ([]string, } publicIPs := []string{} for _, interf := range desc.NetworkInterfaces { - publicIPs = append(publicIPs, *interf.Association.PublicIp) + if interf.Association != nil { + publicIPs = append(publicIPs, *interf.Association.PublicIp) + } } return publicIPs, nil } diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index b39afe02..1049a993 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -17,4 +17,5 @@ type API interface { InspectSecret(ctx context.Context, id string) (docker.Secret, error) ListSecrets(ctx context.Context) ([]docker.Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error + ComposePs(background context.Context, project *Project) error } From be1c65d44189bb15ecb7ea0111d4a1739125ed1e Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 28 May 2020 09:24:21 +0200 Subject: [PATCH 086/205] Get more from DescribeTask Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/api.go | 4 +- ecs/pkg/amazon/api_mock.go | 74 +++++++++---------- ecs/pkg/amazon/convert.go | 2 +- ecs/pkg/amazon/list.go | 66 ++++++++++------- ecs/pkg/amazon/sdk.go | 26 ++++--- .../simple-cloudformation-conversion.golden | 2 +- ...formation-with-overrides-conversion.golden | 2 +- ecs/tests/e2e_deploy_services_test.go | 6 +- 8 files changed, 100 insertions(+), 82 deletions(-) diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go index 9fd599a1..5b058584 100644 --- a/ecs/pkg/amazon/api.go +++ b/ecs/pkg/amazon/api.go @@ -1,11 +1,11 @@ package amazon -//go:generate mockgen -destination=./mock/api.go -package=mock . API +//go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon" -package=amazon . API type API interface { downAPI upAPI logsAPI secretsAPI - psAPI + listAPI } diff --git a/ecs/pkg/amazon/api_mock.go b/ecs/pkg/amazon/api_mock.go index 06ef4fc0..40e9f687 100644 --- a/ecs/pkg/amazon/api_mock.go +++ b/ecs/pkg/amazon/api_mock.go @@ -137,6 +137,26 @@ func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0, arg1) } +// DescribeTasks mocks base method +func (m *MockAPI) DescribeTasks(arg0 context.Context, arg1 string, arg2 ...string) ([]TaskStatus, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeTasks", varargs...) + ret0, _ := ret[0].([]TaskStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeTasks indicates an expected call of DescribeTasks +func (mr *MockAPIMockRecorder) DescribeTasks(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTasks", reflect.TypeOf((*MockAPI)(nil).DescribeTasks), varargs...) +} + // GetDefaultVPC mocks base method func (m *MockAPI) GetDefaultVPC(arg0 context.Context) (string, error) { m.ctrl.T.Helper() @@ -166,35 +186,15 @@ func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1, arg2 interface{}) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1, arg2) } -// GetNetworkInterfaces mocks base method -func (m *MockAPI) GetNetworkInterfaces(arg0 context.Context, arg1 string, arg2 ...string) ([]string, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GetNetworkInterfaces", varargs...) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetNetworkInterfaces indicates an expected call of GetNetworkInterfaces -func (mr *MockAPIMockRecorder) GetNetworkInterfaces(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNetworkInterfaces", reflect.TypeOf((*MockAPI)(nil).GetNetworkInterfaces), varargs...) -} - // GetPublicIPs mocks base method -func (m *MockAPI) GetPublicIPs(arg0 context.Context, arg1 ...string) ([]string, error) { +func (m *MockAPI) GetPublicIPs(arg0 context.Context, arg1 ...string) (map[string]string, error) { m.ctrl.T.Helper() varargs := []interface{}{arg0} for _, a := range arg1 { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetPublicIPs", varargs...) - ret0, _ := ret[0].([]string) + ret0, _ := ret[0].(map[string]string) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -236,21 +236,6 @@ func (mr *MockAPIMockRecorder) GetSubNets(arg0, arg1 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), arg0, arg1) } -// GetTasks mocks base method -func (m *MockAPI) GetTasks(arg0 context.Context, arg1, arg2 string) ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTasks", arg0, arg1, arg2) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTasks indicates an expected call of GetTasks -func (mr *MockAPIMockRecorder) GetTasks(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTasks", reflect.TypeOf((*MockAPI)(nil).GetTasks), arg0, arg1, arg2) -} - // InspectSecret mocks base method func (m *MockAPI) InspectSecret(arg0 context.Context, arg1 string) (docker.Secret, error) { m.ctrl.T.Helper() @@ -281,6 +266,21 @@ func (mr *MockAPIMockRecorder) ListSecrets(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSecrets", reflect.TypeOf((*MockAPI)(nil).ListSecrets), arg0) } +// ListTasks mocks base method +func (m *MockAPI) ListTasks(arg0 context.Context, arg1, arg2 string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListTasks", arg0, arg1, arg2) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListTasks indicates an expected call of ListTasks +func (mr *MockAPIMockRecorder) ListTasks(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTasks", reflect.TypeOf((*MockAPI)(nil).ListTasks), arg0, arg1, arg2) +} + // StackExists mocks base method func (m *MockAPI) StackExists(arg0 context.Context, arg1 string) (bool, error) { m.ctrl.T.Helper() diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 93d546a6..807f23ca 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -77,7 +77,7 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe }, }, Cpu: cpu, - Family: fmt.Sprintf("%s-%s", project.Name, service.Name), + Family: project.Name, IpcMode: service.Ipc, Memory: mem, NetworkMode: ecsapi.NetworkModeAwsvpc, // FIXME could be set by service.NetworkMode, Fargate only supports network mode ‘awsvpc’. diff --git a/ecs/pkg/amazon/list.go b/ecs/pkg/amazon/list.go index d401e0d6..2515fffe 100644 --- a/ecs/pkg/amazon/list.go +++ b/ecs/pkg/amazon/list.go @@ -17,39 +17,51 @@ func (c *client) ComposePs(ctx context.Context, project *compose.Project) error } w := tabwriter.NewWriter(os.Stdout, 20, 2, 3, ' ', 0) fmt.Fprintf(w, "Name\tState\tPorts\n") - for _, s := range project.Services { - tasks, err := c.api.GetTasks(ctx, cluster, s.Name) - if err != nil { - return err - } - if len(tasks) == 0 { - continue - } - // TODO get more data from DescribeTask, including tasks status - networkInterfaces, err := c.api.GetNetworkInterfaces(ctx, cluster, tasks...) - if err != nil { - return err - } - if len(networkInterfaces) == 0 { - fmt.Fprintf(w, "%s\t%s\t\n", s.Name, "Provisioning") - continue - } - publicIps, err := c.api.GetPublicIPs(ctx, networkInterfaces...) - if err != nil { - return err + arns, err := c.api.ListTasks(ctx, cluster, project.Name) + if err != nil { + return err + } + + tasks, err := c.api.DescribeTasks(ctx, cluster, arns...) + if err != nil { + return err + } + + networkInterfaces := []string{} + for _, t := range tasks { + if t.NetworkInterface != "" { + networkInterfaces = append(networkInterfaces, t.NetworkInterface) } + } + publicIps, err := c.api.GetPublicIPs(ctx, networkInterfaces...) + if err != nil { + return err + } + + for _, t := range tasks { ports := []string{} - for _, p := range s.Ports { - ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", strings.Join(publicIps, ","), p.Published, p.Target, p.Protocol)) + s, err := project.GetService(t.Service) + if err != nil { + return err } - fmt.Fprintf(w, "%s\t%s\t%s\n", s.Name, "Up", strings.Join(ports, ", ")) + for _, p := range s.Ports { + ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", publicIps[t.NetworkInterface], p.Published, p.Target, p.Protocol)) + } + fmt.Fprintf(w, "%s\t%s\t%s\n", s.Name, t.State, strings.Join(ports, ", ")) } w.Flush() return nil } -type psAPI interface { - GetTasks(ctx context.Context, cluster string, name string) ([]string, error) - GetNetworkInterfaces(ctx context.Context, cluster string, arns ...string) ([]string, error) - GetPublicIPs(ctx context.Context, interfaces ...string) ([]string, error) +type TaskStatus struct { + State string + Service string + NetworkInterface string + PublicIP string +} + +type listAPI interface { + ListTasks(ctx context.Context, cluster string, name string) ([]string, error) + DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]TaskStatus, error) + GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error) } diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 3f9b9e6b..1580901f 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -341,10 +341,10 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer LogConsumer) err } } -func (s sdk) GetTasks(ctx context.Context, cluster string, name string) ([]string, error) { +func (s sdk) ListTasks(ctx context.Context, cluster string, name string) ([]string, error) { tasks, err := s.ECS.ListTasksWithContext(ctx, &ecs.ListTasksInput{ - Cluster: aws.String(cluster), - ServiceName: aws.String(name), + Cluster: aws.String(cluster), + Family: aws.String(name), }) if err != nil { return nil, err @@ -356,7 +356,7 @@ func (s sdk) GetTasks(ctx context.Context, cluster string, name string) ([]strin return arns, nil } -func (s sdk) GetNetworkInterfaces(ctx context.Context, cluster string, arns ...string) ([]string, error) { +func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]TaskStatus, error) { tasks, err := s.ECS.DescribeTasksWithContext(ctx, &ecs.DescribeTasksInput{ Cluster: aws.String(cluster), Tasks: aws.StringSlice(arns), @@ -364,32 +364,38 @@ func (s sdk) GetNetworkInterfaces(ctx context.Context, cluster string, arns ...s if err != nil { return nil, err } - interfaces := []string{} + result := []TaskStatus{} for _, task := range tasks.Tasks { + var networkInterface string for _, attachement := range task.Attachments { if *attachement.Type == "ElasticNetworkInterface" { for _, pair := range attachement.Details { if *pair.Name == "networkInterfaceId" { - interfaces = append(interfaces, *pair.Value) + networkInterface = *pair.Value } } } } + result = append(result, TaskStatus{ + State: *task.LastStatus, + Service: strings.Replace(*task.Group, "service:", "", 1), + NetworkInterface: networkInterface, + }) } - return interfaces, nil + return result, nil } -func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) ([]string, error) { +func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error) { desc, err := s.EC2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ NetworkInterfaceIds: aws.StringSlice(interfaces), }) if err != nil { return nil, err } - publicIPs := []string{} + publicIPs := map[string]string{} for _, interf := range desc.NetworkInterfaces { if interf.Association != nil { - publicIPs = append(publicIPs, *interf.Association.PublicIp) + publicIPs[*interf.NetworkInterfaceId] = *interf.Association.PublicIp } } return publicIPs, nil diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index d78914e6..677c3483 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -181,7 +181,7 @@ "ExecutionRoleArn": { "Ref": "SimpleTaskExecutionRole" }, - "Family": "TestSimpleConvert-simple", + "Family": "TestSimpleConvert", "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 1eaeed17..b81f6351 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -181,7 +181,7 @@ "ExecutionRoleArn": { "Ref": "SimpleTaskExecutionRole" }, - "Family": "TestSimpleWithOverrides-simple", + "Family": "TestSimpleWithOverrides", "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ diff --git a/ecs/tests/e2e_deploy_services_test.go b/ecs/tests/e2e_deploy_services_test.go index 80d77f31..c7002a19 100644 --- a/ecs/tests/e2e_deploy_services_test.go +++ b/ecs/tests/e2e_deploy_services_test.go @@ -48,10 +48,10 @@ func composeUpSimpleService(t *testing.T, cmd icmd.Cmd, awsContext docker.AwsCon }) assert.NilError(t, err) sdk := amazon.NewAPI(session) - arns, err := sdk.GetTasks(bgContext, t.Name(), "simple") + arns, err := sdk.ListTasks(bgContext, t.Name(), t.Name()) assert.NilError(t, err) - networkInterfaces, err := sdk.GetNetworkInterfaces(bgContext, t.Name(), arns...) - publicIps, err := sdk.GetPublicIPs(context.Background(), networkInterfaces...) + tasks, err := sdk.DescribeTasks(bgContext, t.Name(), arns...) + publicIps, err := sdk.GetPublicIPs(context.Background(), tasks[0].NetworkInterface) assert.NilError(t, err) for _, ip := range publicIps { icmd.RunCommand("curl", "-I", "http://"+ip).Assert(t, icmd.Success) From 6c57fb9693565523853912f397e6d416eb268f7b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 28 May 2020 11:09:36 +0200 Subject: [PATCH 087/205] support deploy.replicas Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 6 +++++- ecs/pkg/amazon/convert.go | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 9df24750..ebd948af 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -167,9 +167,13 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err serviceSecurityGroups = append(serviceSecurityGroups, cloudformation.Ref(logicalName)) } + desiredCount := 1 + if service.Deploy != nil && service.Deploy.Replicas != nil { + desiredCount = int(*service.Deploy.Replicas) + } template.Resources[fmt.Sprintf("%sService", normalizeResourceName(service.Name))] = &ecs.Service{ Cluster: cluster, - DesiredCount: 1, + DesiredCount: desiredCount, LaunchType: ecsapi.LaunchTypeFargate, NetworkConfiguration: &ecs.Service_NetworkConfiguration{ AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 807f23ca..5d74df29 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -317,14 +317,10 @@ func getImage(image string) string { func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name - credential := "" for key, value := range service.Extras { if key == "x-aws-pull_credentials" { - credential = value.(string) + return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: value.(string)} } } - if credential != "" { - return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: credential} - } return nil } From 5783b6355603c3305ea269b887296b167b75a817 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 28 May 2020 16:45:53 +0200 Subject: [PATCH 088/205] Service can freely communicate within a network Signed-off-by: Nicolas De Loof --- ecs/README.md | 8 ++- ecs/pkg/amazon/cloudformation.go | 53 ++++++++++++------- .../simple-cloudformation-conversion.golden | 13 +++++ ...formation-with-overrides-conversion.golden | 13 +++++ ecs/pkg/amazon/up.go | 1 + ecs/pkg/compose/normalize.go | 29 ++++++++++ 6 files changed, 93 insertions(+), 24 deletions(-) diff --git a/ecs/README.md b/ecs/README.md index afa354d3..c0cb834b 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -55,11 +55,9 @@ according to the networks declared in Compose model. Doing so, services attached communicate together, while services from distinct SecurityGroups can't. We just can't set service aliasses per network. A CloudMap private namespace is created for application as `{project}.local`. Services get registered so that we -get service discovery and DNS round-robin (equivalent for Compose's `endpoint_mode: dnsrr`). Hostname-only service -discovery is enabled by running application containers with `LOCALDOMAIN={project}.local` -(see [resolv.conf(5)](http://man7.org/linux/man-pages/man5/resolv.conf.5.html)). This works out-of-the-box for -debian-based Docker images. Alpine images have to include a tiny entrypoint script to replicate this feature: +get service discovery and DNS round-robin (equivalent for Compose's `endpoint_mode: dnsrr`). Docker images SHOULD +include a tiny entrypoint script to replicate this feature: ```shell script -if [ $LOCALDOMAIN ]; then echo "search ${LOCALDOMAIN}" >> /etc/resolv.conf; fi +if [ ! -z LOCALDOMAIN ]; then echo "search ${LOCALDOMAIN}" >> /etc/resolv.conf; fi ``` diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index ebd948af..a9642a6a 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -5,6 +5,8 @@ import ( "regexp" "strings" + "github.com/compose-spec/compose-go/types" + "github.com/sirupsen/logrus" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" @@ -77,9 +79,10 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } cluster := cloudformation.If("CreateCluster", cloudformation.Ref("Cluster"), cloudformation.Ref(ParameterClusterName)) - for net := range project.Networks { - name, resource := convertNetwork(project, net, cloudformation.Ref(ParameterVPCId)) - template.Resources[name] = resource + for _, net := range project.Networks { + for k, v := range convertNetwork(project, net, cloudformation.Ref(ParameterVPCId)) { + template.Resources[k] = v + } } logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) @@ -204,25 +207,28 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return template, nil } -func convertNetwork(project *compose.Project, net string, vpc string) (string, cloudformation.Resource) { +func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc string) map[string]cloudformation.Resource { + resources := map[string]cloudformation.Resource{} var ingresses []ec2.SecurityGroup_Ingress - for _, service := range project.Services { - if _, ok := service.Networks[net]; ok { - for _, port := range service.Ports { - ingresses = append(ingresses, ec2.SecurityGroup_Ingress{ - CidrIp: "0.0.0.0/0", - Description: fmt.Sprintf("%s:%d/%s", service.Name, port.Target, port.Protocol), - FromPort: int(port.Target), - IpProtocol: strings.ToUpper(port.Protocol), - ToPort: int(port.Target), - }) + if !net.Internal { + for _, service := range project.Services { + if _, ok := service.Networks[net.Name]; ok { + for _, port := range service.Ports { + ingresses = append(ingresses, ec2.SecurityGroup_Ingress{ + CidrIp: "0.0.0.0/0", + Description: fmt.Sprintf("%s:%d/%s", service.Name, port.Target, port.Protocol), + FromPort: int(port.Target), + IpProtocol: strings.ToUpper(port.Protocol), + ToPort: int(port.Target), + }) + } } } } - securityGroup := networkResourceName(project, net) - resource := &ec2.SecurityGroup{ - GroupDescription: fmt.Sprintf("%s %s Security Group", project.Name, net), + securityGroup := networkResourceName(project, net.Name) + resources[securityGroup] = &ec2.SecurityGroup{ + GroupDescription: fmt.Sprintf("%s %s Security Group", project.Name, net.Name), GroupName: securityGroup, SecurityGroupIngress: ingresses, VpcId: vpc, @@ -233,11 +239,20 @@ func convertNetwork(project *compose.Project, net string, vpc string) (string, c }, { Key: NetworkTag, - Value: net, + Value: net.Name, }, }, } - return securityGroup, resource + + ingress := securityGroup + "Ingress" + resources[ingress] = &ec2.SecurityGroupIngress{ + Description: fmt.Sprintf("Allow communication within network %s", net.Name), + IpProtocol: "-1", // all protocols + GroupId: cloudformation.Ref(securityGroup), + SourceSecurityGroupId: cloudformation.Ref(securityGroup), + } + + return resources } func networkResourceName(project *compose.Project, network string) string { diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 677c3483..613aba14 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -232,6 +232,19 @@ } }, "Type": "AWS::EC2::SecurityGroup" + }, + "TestSimpleConvertDefaultNetworkIngress": { + "Properties": { + "Description": "Allow communication within network default", + "GroupId": { + "Ref": "TestSimpleConvertDefaultNetwork" + }, + "IpProtocol": "-1", + "SourceSecurityGroupId": { + "Ref": "TestSimpleConvertDefaultNetwork" + } + }, + "Type": "AWS::EC2::SecurityGroupIngress" } } } diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index b81f6351..0e506fce 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -232,6 +232,19 @@ } }, "Type": "AWS::EC2::SecurityGroup" + }, + "TestSimpleWithOverridesDefaultNetworkIngress": { + "Properties": { + "Description": "Allow communication within network default", + "GroupId": { + "Ref": "TestSimpleWithOverridesDefaultNetwork" + }, + "IpProtocol": "-1", + "SourceSecurityGroupId": { + "Ref": "TestSimpleWithOverridesDefaultNetwork" + } + }, + "Type": "AWS::EC2::SecurityGroupIngress" } } } diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index e3297249..23517a11 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -54,6 +54,7 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return err } + fmt.Println() return c.WaitStackCompletion(ctx, project.Name, StackCreate) } diff --git a/ecs/pkg/compose/normalize.go b/ecs/pkg/compose/normalize.go index 0061a1f1..861d146f 100644 --- a/ecs/pkg/compose/normalize.go +++ b/ecs/pkg/compose/normalize.go @@ -49,5 +49,34 @@ func Normalize(model *types.Config) error { } model.Services[i] = s } + + for i, n := range model.Networks { + if n.Name == "" { + n.Name = i + model.Networks[i] = n + } + } + + for i, v := range model.Volumes { + if v.Name == "" { + v.Name = i + model.Volumes[i] = v + } + } + + for i, c := range model.Configs { + if c.Name == "" { + c.Name = i + model.Configs[i] = c + } + } + + for i, s := range model.Secrets { + if s.Name == "" { + s.Name = i + model.Secrets[i] = s + } + } + return nil } From 5080a83242928414098b99e249fae7994ca15b32 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 29 May 2020 08:55:44 +0200 Subject: [PATCH 089/205] prevent "Tasks cannot be empty" error Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/list.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/list.go b/ecs/pkg/amazon/list.go index 2515fffe..0a5f4c17 100644 --- a/ecs/pkg/amazon/list.go +++ b/ecs/pkg/amazon/list.go @@ -17,10 +17,15 @@ func (c *client) ComposePs(ctx context.Context, project *compose.Project) error } w := tabwriter.NewWriter(os.Stdout, 20, 2, 3, ' ', 0) fmt.Fprintf(w, "Name\tState\tPorts\n") + defer w.Flush() + arns, err := c.api.ListTasks(ctx, cluster, project.Name) if err != nil { return err } + if len(arns) == 0 { + return nil + } tasks, err := c.api.DescribeTasks(ctx, cluster, arns...) if err != nil { @@ -49,7 +54,6 @@ func (c *client) ComposePs(ctx context.Context, project *compose.Project) error } fmt.Fprintf(w, "%s\t%s\t%s\n", s.Name, t.State, strings.Join(ports, ", ")) } - w.Flush() return nil } From ff882903023e152b31ccceb9f89b812d52d30c78 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 29 May 2020 15:13:16 +0200 Subject: [PATCH 090/205] Make `ps` order predictable so one can run `watch docker ecs compose ps` Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/list.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ecs/pkg/amazon/list.go b/ecs/pkg/amazon/list.go index 0a5f4c17..4b16f98c 100644 --- a/ecs/pkg/amazon/list.go +++ b/ecs/pkg/amazon/list.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "sort" "strings" "text/tabwriter" @@ -43,6 +44,10 @@ func (c *client) ComposePs(ctx context.Context, project *compose.Project) error return err } + sort.Slice(tasks, func(i, j int) bool { + return strings.Compare(tasks[i].Service, tasks[j].Service) < 0 + }) + for _, t := range tasks { ports := []string{} s, err := project.GetService(t.Service) From 7d4222a725f592dfffac6cce79faf699a97f6fd6 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 2 Jun 2020 13:54:33 +0200 Subject: [PATCH 091/205] Implement depends_on using CloudFormation Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/check.go | 1 - ecs/pkg/amazon/cloudformation.go | 18 ++++++++++++++---- ecs/pkg/amazon/compatibility.go | 8 -------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ecs/pkg/amazon/check.go b/ecs/pkg/amazon/check.go index d249afe4..169794c7 100644 --- a/ecs/pkg/amazon/check.go +++ b/ecs/pkg/amazon/check.go @@ -11,7 +11,6 @@ type Warnings []string type CompatibilityChecker interface { CheckService(service *types.ServiceConfig) CheckCapAdd(service *types.ServiceConfig) - CheckDependsOn(service *types.ServiceConfig) CheckDNS(service *types.ServiceConfig) CheckDNSOpts(service *types.ServiceConfig) CheckDNSSearch(service *types.ServiceConfig) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index a9642a6a..87d9f060 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -174,10 +174,16 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err if service.Deploy != nil && service.Deploy.Replicas != nil { desiredCount = int(*service.Deploy.Replicas) } - template.Resources[fmt.Sprintf("%sService", normalizeResourceName(service.Name))] = &ecs.Service{ - Cluster: cluster, - DesiredCount: desiredCount, - LaunchType: ecsapi.LaunchTypeFargate, + + dependsOn := []string{} + for _, dependency := range service.DependsOn { + dependsOn = append(dependsOn, serviceResourceName(dependency)) + } + template.Resources[serviceResourceName(service.Name)] = &ecs.Service{ + AWSCloudFormationDependsOn: dependsOn, + Cluster: cluster, + DesiredCount: desiredCount, + LaunchType: ecsapi.LaunchTypeFargate, NetworkConfiguration: &ecs.Service_NetworkConfiguration{ AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: ecsapi.AssignPublicIpEnabled, @@ -259,6 +265,10 @@ func networkResourceName(project *compose.Project, network string) string { return fmt.Sprintf("%s%sNetwork", normalizeResourceName(project.Name), normalizeResourceName(network)) } +func serviceResourceName(dependency string) string { + return fmt.Sprintf("%sService", normalizeResourceName(dependency)) +} + func normalizeResourceName(s string) string { return strings.Title(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(s, "")) } diff --git a/ecs/pkg/amazon/compatibility.go b/ecs/pkg/amazon/compatibility.go index 6049a4d4..1a7f136a 100644 --- a/ecs/pkg/amazon/compatibility.go +++ b/ecs/pkg/amazon/compatibility.go @@ -21,7 +21,6 @@ func (c *FargateCompatibilityChecker) Errors() []error { func (c *FargateCompatibilityChecker) CheckService(service *types.ServiceConfig) { c.CheckCapAdd(service) - c.CheckDependsOn(service) c.CheckDNS(service) c.CheckDNSOpts(service) c.CheckDNSSearch(service) @@ -47,13 +46,6 @@ func (c *FargateCompatibilityChecker) CheckNetworkMode(service *types.ServiceCon service.NetworkMode = ecs.NetworkModeAwsvpc } -func (c *FargateCompatibilityChecker) CheckDependsOn(service *types.ServiceConfig) { - if len(service.DependsOn) != 0 { - c.error("'depends_on' is not supported") - service.DependsOn = nil - } -} - func (c *FargateCompatibilityChecker) CheckLinks(service *types.ServiceConfig) { if len(service.Links) != 0 { c.error("'links' is not supported") From 1bf4bc9d4614efa819d9fcc832d8c86a9844a57b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 2 Jun 2020 14:17:48 +0200 Subject: [PATCH 092/205] Use distinct family per service definition Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/convert.go | 2 +- ecs/pkg/amazon/list.go | 10 +++++++--- ecs/pkg/amazon/sdk.go | 6 +++--- .../simple/simple-cloudformation-conversion.golden | 2 +- ...ple-cloudformation-with-overrides-conversion.golden | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 5d74df29..8f43d62f 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -77,7 +77,7 @@ func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDe }, }, Cpu: cpu, - Family: project.Name, + Family: fmt.Sprintf("%s-%s", project.Name, service.Name), IpcMode: service.Ipc, Memory: mem, NetworkMode: ecsapi.NetworkModeAwsvpc, // FIXME could be set by service.NetworkMode, Fargate only supports network mode ‘awsvpc’. diff --git a/ecs/pkg/amazon/list.go b/ecs/pkg/amazon/list.go index 4b16f98c..98903f40 100644 --- a/ecs/pkg/amazon/list.go +++ b/ecs/pkg/amazon/list.go @@ -20,9 +20,13 @@ func (c *client) ComposePs(ctx context.Context, project *compose.Project) error fmt.Fprintf(w, "Name\tState\tPorts\n") defer w.Flush() - arns, err := c.api.ListTasks(ctx, cluster, project.Name) - if err != nil { - return err + arns := []string{} + for _, service := range project.Services { + tasks, err := c.api.ListTasks(ctx, cluster, service.Name) + if err != nil { + return err + } + arns = append(arns, tasks...) } if len(arns) == 0 { return nil diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 1580901f..6bc85381 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -341,10 +341,10 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer LogConsumer) err } } -func (s sdk) ListTasks(ctx context.Context, cluster string, name string) ([]string, error) { +func (s sdk) ListTasks(ctx context.Context, cluster string, service string) ([]string, error) { tasks, err := s.ECS.ListTasksWithContext(ctx, &ecs.ListTasksInput{ - Cluster: aws.String(cluster), - Family: aws.String(name), + Cluster: aws.String(cluster), + ServiceName: aws.String(service), }) if err != nil { return nil, err diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 613aba14..c1233d0a 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -181,7 +181,7 @@ "ExecutionRoleArn": { "Ref": "SimpleTaskExecutionRole" }, - "Family": "TestSimpleConvert", + "Family": "TestSimpleConvert-simple", "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 0e506fce..ca03c117 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -181,7 +181,7 @@ "ExecutionRoleArn": { "Ref": "SimpleTaskExecutionRole" }, - "Family": "TestSimpleWithOverrides", + "Family": "TestSimpleWithOverrides-simple", "Memory": "512", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ From b702065075b07f3ec7e52dc0604badae829e0628 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 3 Jun 2020 14:31:19 +0200 Subject: [PATCH 093/205] custom extension to select existing VPC and SecurityGroups Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 26 ++++++++++--------- ecs/pkg/amazon/convert.go | 2 +- .../simple-cloudformation-conversion.golden | 4 +-- ...formation-with-overrides-conversion.golden | 4 +-- ecs/pkg/amazon/up.go | 21 +++++++-------- ecs/pkg/amazon/x.go | 7 +++++ 6 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 ecs/pkg/amazon/x.go diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 87d9f060..84d0b767 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -57,11 +57,11 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err */ template.Parameters[ParameterSubnet1Id] = cloudformation.Parameter{ Type: "AWS::EC2::Subnet::Id", - Description: "SubnetId,for Availability Zone 1 in the region in your VPC", + Description: "SubnetId, for Availability Zone 1 in the region in your VPC", } template.Parameters[ParameterSubnet2Id] = cloudformation.Parameter{ Type: "AWS::EC2::Subnet::Id", - Description: "SubnetId,for Availability Zone 1 in the region in your VPC", + Description: "SubnetId, for Availability Zone 2 in the region in your VPC", } // Create Cluster is `ParameterClusterName` parameter is not set @@ -79,10 +79,9 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } cluster := cloudformation.If("CreateCluster", cloudformation.Ref("Cluster"), cloudformation.Ref(ParameterClusterName)) + networks := map[string]string{} for _, net := range project.Networks { - for k, v := range convertNetwork(project, net, cloudformation.Ref(ParameterVPCId)) { - template.Resources[k] = v - } + networks[net.Name] = convertNetwork(project, net, cloudformation.Ref(ParameterVPCId), template) } logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) @@ -166,8 +165,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err serviceSecurityGroups := []string{} for net := range service.Networks { - logicalName := networkResourceName(project, net) - serviceSecurityGroups = append(serviceSecurityGroups, cloudformation.Ref(logicalName)) + serviceSecurityGroups = append(serviceSecurityGroups, networks[net]) } desiredCount := 1 @@ -213,8 +211,12 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return template, nil } -func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc string) map[string]cloudformation.Resource { - resources := map[string]cloudformation.Resource{} +func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc string, template *cloudformation.Template) string { + if sg, ok := net.Extras[ExtensionSecurityGroup]; ok { + logrus.Debugf("Security Group for network %q set by user to %q", net.Name, sg) + return sg.(string) + } + var ingresses []ec2.SecurityGroup_Ingress if !net.Internal { for _, service := range project.Services { @@ -233,7 +235,7 @@ func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc strin } securityGroup := networkResourceName(project, net.Name) - resources[securityGroup] = &ec2.SecurityGroup{ + template.Resources[securityGroup] = &ec2.SecurityGroup{ GroupDescription: fmt.Sprintf("%s %s Security Group", project.Name, net.Name), GroupName: securityGroup, SecurityGroupIngress: ingresses, @@ -251,14 +253,14 @@ func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc strin } ingress := securityGroup + "Ingress" - resources[ingress] = &ec2.SecurityGroupIngress{ + template.Resources[ingress] = &ec2.SecurityGroupIngress{ Description: fmt.Sprintf("Allow communication within network %s", net.Name), IpProtocol: "-1", // all protocols GroupId: cloudformation.Ref(securityGroup), SourceSecurityGroupId: cloudformation.Ref(securityGroup), } - return resources + return cloudformation.Ref(securityGroup) } func networkResourceName(project *compose.Project, network string) string { diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/convert.go index 8f43d62f..82047ddb 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/convert.go @@ -318,7 +318,7 @@ func getImage(image string) string { func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name for key, value := range service.Extras { - if key == "x-aws-pull_credentials" { + if key == ExtensionPullCredentials { return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: value.(string)} } } diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index c1233d0a..8a95637a 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -16,11 +16,11 @@ "Type": "String" }, "ParameterSubnet1Id": { - "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", + "Description": "SubnetId, for Availability Zone 1 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" }, "ParameterSubnet2Id": { - "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", + "Description": "SubnetId, for Availability Zone 2 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" }, "ParameterVPCId": { diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index ca03c117..a5551ddc 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -16,11 +16,11 @@ "Type": "String" }, "ParameterSubnet1Id": { - "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", + "Description": "SubnetId, for Availability Zone 1 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" }, "ParameterSubnet2Id": { - "Description": "SubnetId,for Availability Zone 1 in the region in your VPC", + "Description": "SubnetId, for Availability Zone 2 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" }, "ParameterVPCId": { diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 23517a11..21500c38 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -59,18 +59,15 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error } func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, error) { - //check compose file for the default external network - if net, ok := project.Networks["default"]; ok { - if net.External.External { - vpc := net.Name - ok, err := c.api.VpcExists(ctx, vpc) - if err != nil { - return "", err - } - if !ok { - return "", fmt.Errorf("VPC does not exist: %s", vpc) - } - return vpc, nil + //check compose file for custom VPC selected + if vpc, ok := project.Extras[ExtensionVPC]; ok { + vpcID := vpc.(string) + ok, err := c.api.VpcExists(ctx, vpcID) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("VPC does not exist: %s", vpc) } } defaultVPC, err := c.api.GetDefaultVPC(ctx) diff --git a/ecs/pkg/amazon/x.go b/ecs/pkg/amazon/x.go new file mode 100644 index 00000000..0b022bac --- /dev/null +++ b/ecs/pkg/amazon/x.go @@ -0,0 +1,7 @@ +package amazon + +const ( + ExtensionSecurityGroup = "x-aws-securitygroup" + ExtensionVPC = "x-aws-vpc" + ExtensionPullCredentials = "x-aws-pull_credentials" +) From fc9b10fc9118c0f283de3938fdcae0f2c82e350e Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 3 Jun 2020 11:11:57 +0200 Subject: [PATCH 094/205] add load balancer Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 119 ++++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 8 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 84d0b767..400862f5 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -9,12 +9,14 @@ import ( "github.com/sirupsen/logrus" + "github.com/aws/aws-sdk-go/service/elbv2" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" + "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/awslabs/goformation/v4/cloudformation/logs" cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" @@ -144,13 +146,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), } - if len(service.Ports) > 0 { - records = append(records, cloudmap.Service_DnsRecord{ - TTL: 60, - Type: cloudmapapi.RecordTypeSrv, - }) - serviceRegistry.Port = int(service.Ports[0].Target) - } + loadBalancers := []ecs.Service_LoadBalancer{} template.Resources[serviceRegistration] = &cloudmap.Service{ Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), @@ -168,12 +164,118 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err serviceSecurityGroups = append(serviceSecurityGroups, networks[net]) } + dependsOn := []string{} + if len(service.Ports) > 0 { + records = append(records, cloudmap.Service_DnsRecord{ + TTL: 60, + Type: cloudmapapi.RecordTypeSrv, + }) + //serviceRegistry.Port = int(service.Ports[0].Target) + // add targetgroup for each published port + for _, port := range service.Ports { + targetGroupName := fmt.Sprintf( + "%s%s%sTargetGroup", + normalizeResourceName(service.Name), + strings.ToUpper(port.Protocol), + string(port.Published), + ) + listenerName := fmt.Sprintf( + "%s%s%sListener", + normalizeResourceName(service.Name), + strings.ToUpper(port.Protocol), + string(port.Published), + ) + loadBalancerName := fmt.Sprintf( + "%s%s%sLoadBalancer", + normalizeResourceName(service.Name), + strings.ToUpper(port.Protocol), + string(port.Published), + ) + dependsOn = append(dependsOn, listenerName) + lbType := "network" + lbSecGroups := []string{} + protocolType := strings.ToUpper(port.Protocol) + targetType := elbv2.TargetTypeEnumInstance + if port.Published == 80 || port.Published == 443 { + lbType = "application" + lbSecGroups = serviceSecurityGroups + protocolType = "HTTPS" + targetType = elbv2.TargetTypeEnumIp + if port.Published == 80 { + protocolType = "HTTP" + } + } + + template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{ + Name: targetGroupName, + Port: int(port.Target), + Protocol: protocolType, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + { + Key: ServiceTag, + Value: service.Name, + }, + }, + VpcId: cloudformation.Ref(ParameterVPCId), + TargetType: targetType, + } + + template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ + Name: loadBalancerName, + Scheme: "internet-facing", + SecurityGroups: lbSecGroups, + Subnets: []string{ + cloudformation.Ref(ParameterSubnet1Id), + cloudformation.Ref(ParameterSubnet2Id), + }, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + { + Key: ServiceTag, + Value: service.Name, + }, + }, + Type: lbType, + } + + template.Resources[listenerName] = &elasticloadbalancingv2.Listener{ + DefaultActions: []elasticloadbalancingv2.Listener_Action{ + { + ForwardConfig: &elasticloadbalancingv2.Listener_ForwardConfig{ + TargetGroups: []elasticloadbalancingv2.Listener_TargetGroupTuple{ + { + TargetGroupArn: cloudformation.Ref(targetGroupName), + }, + }, + }, + Type: elbv2.ActionTypeEnumForward, + }, + }, + LoadBalancerArn: cloudformation.Ref(loadBalancerName), + Protocol: protocolType, + Port: int(port.Published), + } + + loadBalancers = append(loadBalancers, ecs.Service_LoadBalancer{ + ContainerName: service.Name, + ContainerPort: int(port.Published), + TargetGroupArn: cloudformation.Ref(targetGroupName), + }) + } + } + desiredCount := 1 if service.Deploy != nil && service.Deploy.Replicas != nil { desiredCount = int(*service.Deploy.Replicas) } - dependsOn := []string{} for _, dependency := range service.DependsOn { dependsOn = append(dependsOn, serviceResourceName(dependency)) } @@ -182,6 +284,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Cluster: cluster, DesiredCount: desiredCount, LaunchType: ecsapi.LaunchTypeFargate, + LoadBalancers: loadBalancers, NetworkConfiguration: &ecs.Service_NetworkConfiguration{ AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: ecsapi.AssignPublicIpEnabled, From ae3101fe1237e50955c7efa2e6a531494eeac2a5 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 3 Jun 2020 13:06:21 +0200 Subject: [PATCH 095/205] create unique load balancer per app and cleanup Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 118 ++++++++++++++----------------- 1 file changed, 55 insertions(+), 63 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 400862f5..4c57a7c7 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -136,17 +136,15 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", normalizeResourceName(service.Name)) - records := []cloudmap.Service_DnsRecord{ - { - TTL: 60, - Type: cloudmapapi.RecordTypeA, - }, - } serviceRegistry := ecs.Service_ServiceRegistry{ RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), } - loadBalancers := []ecs.Service_LoadBalancer{} + serviceSecurityGroups := []string{} + for net := range service.Networks { + logicalName := networkResourceName(project, net) + serviceSecurityGroups = append(serviceSecurityGroups, cloudformation.Ref(logicalName)) + } template.Resources[serviceRegistration] = &cloudmap.Service{ Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), @@ -154,7 +152,12 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Name: service.Name, NamespaceId: cloudformation.Ref("CloudMap"), DnsConfig: &cloudmap.Service_DnsConfig{ - DnsRecords: records, + DnsRecords: []cloudmap.Service_DnsRecord{ + { + TTL: 60, + Type: cloudmapapi.RecordTypeA, + }, + }, RoutingPolicy: cloudmapapi.RoutingPolicyMultivalue, }, } @@ -165,47 +168,55 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } dependsOn := []string{} + loadBalancers := []ecs.Service_LoadBalancer{} if len(service.Ports) > 0 { - records = append(records, cloudmap.Service_DnsRecord{ - TTL: 60, - Type: cloudmapapi.RecordTypeSrv, - }) - //serviceRegistry.Port = int(service.Ports[0].Target) - // add targetgroup for each published port for _, port := range service.Ports { - targetGroupName := fmt.Sprintf( - "%s%s%sTargetGroup", - normalizeResourceName(service.Name), - strings.ToUpper(port.Protocol), - string(port.Published), - ) - listenerName := fmt.Sprintf( - "%s%s%sListener", - normalizeResourceName(service.Name), - strings.ToUpper(port.Protocol), - string(port.Published), - ) - loadBalancerName := fmt.Sprintf( - "%s%s%sLoadBalancer", - normalizeResourceName(service.Name), - strings.ToUpper(port.Protocol), - string(port.Published), - ) - dependsOn = append(dependsOn, listenerName) - lbType := "network" - lbSecGroups := []string{} + loadBalancerType := "network" + protocolType := strings.ToUpper(port.Protocol) targetType := elbv2.TargetTypeEnumInstance + loadBalancerSecGroups := []string{} + if port.Published == 80 || port.Published == 443 { - lbType = "application" - lbSecGroups = serviceSecurityGroups + loadBalancerType = "application" + loadBalancerSecGroups = serviceSecurityGroups protocolType = "HTTPS" targetType = elbv2.TargetTypeEnumIp if port.Published == 80 { protocolType = "HTTP" } } + loadBalancerName := fmt.Sprintf( + "%s%sLB", + strings.Title(project.Name), + strings.ToUpper(loadBalancerType[0:1]), + ) + // create load baalncer if it doesn't exist + if _, ok := template.Resources[loadBalancerName]; !ok { + template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ + Name: loadBalancerName, + Scheme: "internet-facing", + SecurityGroups: loadBalancerSecGroups, + Subnets: []string{ + cloudformation.Ref(ParameterSubnet1Id), + cloudformation.Ref(ParameterSubnet2Id), + }, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + }, + Type: loadBalancerType, + } + } + targetGroupName := fmt.Sprintf( + "%s%s%sTargetGroup", + normalizeResourceName(service.Name), + strings.ToUpper(port.Protocol), + string(port.Published), + ) template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{ Name: targetGroupName, Port: int(port.Target), @@ -215,36 +226,17 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Key: ProjectTag, Value: project.Name, }, - { - Key: ServiceTag, - Value: service.Name, - }, }, VpcId: cloudformation.Ref(ParameterVPCId), TargetType: targetType, } - - template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ - Name: loadBalancerName, - Scheme: "internet-facing", - SecurityGroups: lbSecGroups, - Subnets: []string{ - cloudformation.Ref(ParameterSubnet1Id), - cloudformation.Ref(ParameterSubnet2Id), - }, - Tags: []tags.Tag{ - { - Key: ProjectTag, - Value: project.Name, - }, - { - Key: ServiceTag, - Value: service.Name, - }, - }, - Type: lbType, - } - + listenerName := fmt.Sprintf( + "%s%s%sListener", + normalizeResourceName(service.Name), + strings.ToUpper(port.Protocol), + string(port.Published), + ) + dependsOn = append(dependsOn, listenerName) template.Resources[listenerName] = &elasticloadbalancingv2.Listener{ DefaultActions: []elasticloadbalancingv2.Listener_Action{ { From 92173eaf352f8c3eb2ad6d064048a2af54dfbed6 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 3 Jun 2020 16:56:13 +0200 Subject: [PATCH 096/205] add SO link for issue if listener is not in service dependencies Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 4c57a7c7..a2df3f8e 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -236,6 +236,8 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err strings.ToUpper(port.Protocol), string(port.Published), ) + //add listener to dependsOn + //https://stackoverflow.com/questions/53971873/the-target-group-does-not-have-an-associated-load-balancer dependsOn = append(dependsOn, listenerName) template.Resources[listenerName] = &elasticloadbalancingv2.Listener{ DefaultActions: []elasticloadbalancingv2.Listener_Action{ From e7f77ca3ef7e58583dfa2d378b439434f9caa3d2 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 3 Jun 2020 17:40:39 +0200 Subject: [PATCH 097/205] add all service security groups to LB Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 58 ++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index a2df3f8e..d66bda50 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -97,6 +97,8 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Name: fmt.Sprintf("%s.local", project.Name), Vpc: cloudformation.Ref(ParameterVPCId), } + //map LB type to security groups list + loadBalancers := map[string][]string{} for _, service := range project.Services { definition, err := Convert(project, service) @@ -168,7 +170,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } dependsOn := []string{} - loadBalancers := []ecs.Service_LoadBalancer{} + serviceLB := []ecs.Service_LoadBalancer{} if len(service.Ports) > 0 { for _, port := range service.Ports { loadBalancerType := "network" @@ -191,26 +193,12 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err strings.Title(project.Name), strings.ToUpper(loadBalancerType[0:1]), ) - // create load baalncer if it doesn't exist - if _, ok := template.Resources[loadBalancerName]; !ok { - - template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ - Name: loadBalancerName, - Scheme: "internet-facing", - SecurityGroups: loadBalancerSecGroups, - Subnets: []string{ - cloudformation.Ref(ParameterSubnet1Id), - cloudformation.Ref(ParameterSubnet2Id), - }, - Tags: []tags.Tag{ - { - Key: ProjectTag, - Value: project.Name, - }, - }, - Type: loadBalancerType, - } + // create load balancer if it doesn't exist + if _, ok := loadBalancers[loadBalancerType]; !ok { + loadBalancers[loadBalancerType] = []string{} } + loadBalancers[loadBalancerType] = append(loadBalancers[loadBalancerType], loadBalancerSecGroups...) + targetGroupName := fmt.Sprintf( "%s%s%sTargetGroup", normalizeResourceName(service.Name), @@ -257,7 +245,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Port: int(port.Published), } - loadBalancers = append(loadBalancers, ecs.Service_LoadBalancer{ + serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ ContainerName: service.Name, ContainerPort: int(port.Published), TargetGroupArn: cloudformation.Ref(targetGroupName), @@ -278,7 +266,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Cluster: cluster, DesiredCount: desiredCount, LaunchType: ecsapi.LaunchTypeFargate, - LoadBalancers: loadBalancers, + LoadBalancers: serviceLB, NetworkConfiguration: &ecs.Service_NetworkConfiguration{ AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: ecsapi.AssignPublicIpEnabled, @@ -305,6 +293,32 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err TaskDefinition: cloudformation.Ref(normalizeResourceName(taskDefinition)), } } + + // create LBs + for lbType, lbSecGroups := range loadBalancers { + loadBalancerName := fmt.Sprintf( + "%s%sLB", + strings.Title(project.Name), + strings.ToUpper(lbType[0:1]), + ) + + template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ + Name: loadBalancerName, + Scheme: "internet-facing", + SecurityGroups: lbSecGroups, + Subnets: []string{ + cloudformation.Ref(ParameterSubnet1Id), + cloudformation.Ref(ParameterSubnet2Id), + }, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + }, + Type: lbType, + } + } return template, nil } From 335806a17961eab352d6a33267b80b718c6e70a1 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 4 Jun 2020 10:26:15 +0200 Subject: [PATCH 098/205] create only one global load balancer - error out if exports port require different types Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 72 ++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index d66bda50..4da59d53 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -97,8 +97,8 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Name: fmt.Sprintf("%s.local", project.Name), Vpc: cloudformation.Ref(ParameterVPCId), } - //map LB type to security groups list - loadBalancers := map[string][]string{} + + var loadBalancer *elasticloadbalancingv2.LoadBalancer for _, service := range project.Services { definition, err := Convert(project, service) @@ -193,11 +193,33 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err strings.Title(project.Name), strings.ToUpper(loadBalancerType[0:1]), ) - // create load balancer if it doesn't exist - if _, ok := loadBalancers[loadBalancerType]; !ok { - loadBalancers[loadBalancerType] = []string{} + // create load balancer if it doesn't exist -- global load balancer + if loadBalancer == nil { + + loadBalancer = &elasticloadbalancingv2.LoadBalancer{ + Name: loadBalancerName, + Scheme: "internet-facing", + SecurityGroups: loadBalancerSecGroups, + Subnets: []string{ + cloudformation.Ref(ParameterSubnet1Id), + cloudformation.Ref(ParameterSubnet2Id), + }, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + }, + Type: loadBalancerType, + } + template.Resources[loadBalancerName] = loadBalancer } - loadBalancers[loadBalancerType] = append(loadBalancers[loadBalancerType], loadBalancerSecGroups...) + if loadBalancer.Type != loadBalancerType { + return nil, fmt.Errorf( + "exposed ports require different types of load balancers, only one type is permitted") + } + loadBalancer.SecurityGroups = append(loadBalancer.SecurityGroups, loadBalancerSecGroups...) + loadBalancer.SecurityGroups = uniqueLBSecurityGroups(loadBalancer.SecurityGroups) targetGroupName := fmt.Sprintf( "%s%s%sTargetGroup", @@ -293,32 +315,6 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err TaskDefinition: cloudformation.Ref(normalizeResourceName(taskDefinition)), } } - - // create LBs - for lbType, lbSecGroups := range loadBalancers { - loadBalancerName := fmt.Sprintf( - "%s%sLB", - strings.Title(project.Name), - strings.ToUpper(lbType[0:1]), - ) - - template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ - Name: loadBalancerName, - Scheme: "internet-facing", - SecurityGroups: lbSecGroups, - Subnets: []string{ - cloudformation.Ref(ParameterSubnet1Id), - cloudformation.Ref(ParameterSubnet2Id), - }, - Tags: []tags.Tag{ - { - Key: ProjectTag, - Value: project.Name, - }, - }, - Type: lbType, - } - } return template, nil } @@ -412,3 +408,15 @@ func (c client) getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) } return nil, nil } + +func uniqueLBSecurityGroups(groups []string) []string { + keys := make(map[string]bool) + unique := []string{} + for _, k := range groups { + if _, val := keys[k]; !val { + keys[k] = true + unique = append(unique, k) + } + } + return unique +} From eddaa70a9efb9aa3befc951eb84041b66fc6bd74 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 4 Jun 2020 11:17:35 +0200 Subject: [PATCH 099/205] create NLB load balancer only Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 89 +++++++++----------------------- 1 file changed, 23 insertions(+), 66 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 4da59d53..98970753 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -98,7 +98,28 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Vpc: cloudformation.Ref(ParameterVPCId), } - var loadBalancer *elasticloadbalancingv2.LoadBalancer + loadBalancerType := "network" + loadBalancerName := fmt.Sprintf( + "%s%sLB", + strings.Title(project.Name), + strings.ToUpper(loadBalancerType[0:1]), + ) + loadBalancer := &elasticloadbalancingv2.LoadBalancer{ + Name: loadBalancerName, + Scheme: "internet-facing", + Subnets: []string{ + cloudformation.Ref(ParameterSubnet1Id), + cloudformation.Ref(ParameterSubnet2Id), + }, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + }, + Type: loadBalancerType, + } + template.Resources[loadBalancerName] = loadBalancer for _, service := range project.Services { definition, err := Convert(project, service) @@ -142,12 +163,6 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), } - serviceSecurityGroups := []string{} - for net := range service.Networks { - logicalName := networkResourceName(project, net) - serviceSecurityGroups = append(serviceSecurityGroups, cloudformation.Ref(logicalName)) - } - template.Resources[serviceRegistration] = &cloudmap.Service{ Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), HealthCheckConfig: healthCheck, @@ -173,54 +188,8 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err serviceLB := []ecs.Service_LoadBalancer{} if len(service.Ports) > 0 { for _, port := range service.Ports { - loadBalancerType := "network" protocolType := strings.ToUpper(port.Protocol) - targetType := elbv2.TargetTypeEnumInstance - loadBalancerSecGroups := []string{} - - if port.Published == 80 || port.Published == 443 { - loadBalancerType = "application" - loadBalancerSecGroups = serviceSecurityGroups - protocolType = "HTTPS" - targetType = elbv2.TargetTypeEnumIp - if port.Published == 80 { - protocolType = "HTTP" - } - } - loadBalancerName := fmt.Sprintf( - "%s%sLB", - strings.Title(project.Name), - strings.ToUpper(loadBalancerType[0:1]), - ) - // create load balancer if it doesn't exist -- global load balancer - if loadBalancer == nil { - - loadBalancer = &elasticloadbalancingv2.LoadBalancer{ - Name: loadBalancerName, - Scheme: "internet-facing", - SecurityGroups: loadBalancerSecGroups, - Subnets: []string{ - cloudformation.Ref(ParameterSubnet1Id), - cloudformation.Ref(ParameterSubnet2Id), - }, - Tags: []tags.Tag{ - { - Key: ProjectTag, - Value: project.Name, - }, - }, - Type: loadBalancerType, - } - template.Resources[loadBalancerName] = loadBalancer - } - if loadBalancer.Type != loadBalancerType { - return nil, fmt.Errorf( - "exposed ports require different types of load balancers, only one type is permitted") - } - loadBalancer.SecurityGroups = append(loadBalancer.SecurityGroups, loadBalancerSecGroups...) - loadBalancer.SecurityGroups = uniqueLBSecurityGroups(loadBalancer.SecurityGroups) - targetGroupName := fmt.Sprintf( "%s%s%sTargetGroup", normalizeResourceName(service.Name), @@ -238,7 +207,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err }, }, VpcId: cloudformation.Ref(ParameterVPCId), - TargetType: targetType, + TargetType: elbv2.TargetTypeEnumIp, } listenerName := fmt.Sprintf( "%s%s%sListener", @@ -408,15 +377,3 @@ func (c client) getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) } return nil, nil } - -func uniqueLBSecurityGroups(groups []string) []string { - keys := make(map[string]bool) - unique := []string{} - for _, k := range groups { - if _, val := keys[k]; !val { - keys[k] = true - unique = append(unique, k) - } - } - return unique -} From f71109be9e24d1231887ac4af7827fbd60ca4bb3 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 4 Jun 2020 11:39:47 +0200 Subject: [PATCH 100/205] update testdata Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- .../simple-cloudformation-conversion.golden | 22 +++++++++++++++++++ ...formation-with-overrides-conversion.golden | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 8a95637a..258fa960 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -245,6 +245,28 @@ } }, "Type": "AWS::EC2::SecurityGroupIngress" + }, + "TestSimpleConvertNLB": { + "Properties": { + "Name": "TestSimpleConvertNLB", + "Scheme": "internet-facing", + "Subnets": [ + { + "Ref": "ParameterSubnet1Id" + }, + { + "Ref": "ParameterSubnet2Id" + } + ], + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleConvert" + } + ], + "Type": "network" + }, + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" } } } diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index a5551ddc..991f65b6 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -245,6 +245,28 @@ } }, "Type": "AWS::EC2::SecurityGroupIngress" + }, + "TestSimpleWithOverridesNLB": { + "Properties": { + "Name": "TestSimpleWithOverridesNLB", + "Scheme": "internet-facing", + "Subnets": [ + { + "Ref": "ParameterSubnet1Id" + }, + { + "Ref": "ParameterSubnet2Id" + } + ], + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleWithOverrides" + } + ], + "Type": "network" + }, + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" } } } From 37177e6d7a63c6dc69e326674f5c0ef5793691b5 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 4 Jun 2020 16:10:34 +0200 Subject: [PATCH 101/205] Split long `Convert` func into smaller, focussed sub-func Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 284 +++++++++++++++++-------------- 1 file changed, 159 insertions(+), 125 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 98970753..0e6966bb 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -69,17 +69,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err // Create Cluster is `ParameterClusterName` parameter is not set template.Conditions["CreateCluster"] = cloudformation.Equals("", cloudformation.Ref(ParameterClusterName)) - template.Resources["Cluster"] = &ecs.Cluster{ - ClusterName: project.Name, - Tags: []tags.Tag{ - { - Key: ProjectTag, - Value: project.Name, - }, - }, - AWSCloudFormationCondition: "CreateCluster", - } - cluster := cloudformation.If("CreateCluster", cloudformation.Ref("Cluster"), cloudformation.Ref(ParameterClusterName)) + cluster := c.createCluster(project, template) networks := map[string]string{} for _, net := range project.Networks { @@ -92,34 +82,8 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } // Private DNS namespace will allow DNS name for the services to be ..local - template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ - Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), - Name: fmt.Sprintf("%s.local", project.Name), - Vpc: cloudformation.Ref(ParameterVPCId), - } - - loadBalancerType := "network" - loadBalancerName := fmt.Sprintf( - "%s%sLB", - strings.Title(project.Name), - strings.ToUpper(loadBalancerType[0:1]), - ) - loadBalancer := &elasticloadbalancingv2.LoadBalancer{ - Name: loadBalancerName, - Scheme: "internet-facing", - Subnets: []string{ - cloudformation.Ref(ParameterSubnet1Id), - cloudformation.Ref(ParameterSubnet2Id), - }, - Tags: []tags.Tag{ - { - Key: ProjectTag, - Value: project.Name, - }, - }, - Type: loadBalancerType, - } - template.Resources[loadBalancerName] = loadBalancer + c.createCloudMap(project, template) + loadBalancer := c.createLoadBalancer(project, template) for _, service := range project.Services { definition, err := Convert(project, service) @@ -127,30 +91,13 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return nil, err } - taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", normalizeResourceName(service.Name)) - policy, err := c.getPolicy(definition) + taskExecutionRole, err := c.createTaskExecutionRole(service, err, definition, template) if err != nil { - return nil, err - } - rolePolicies := []iam.Role_Policy{} - if policy != nil { - rolePolicies = append(rolePolicies, iam.Role_Policy{ - PolicyDocument: policy, - PolicyName: fmt.Sprintf("%sGrantAccessToSecrets", service.Name), - }) - + return template, err } definition.ExecutionRoleArn = cloudformation.Ref(taskExecutionRole) taskDefinition := fmt.Sprintf("%sTaskDefinition", normalizeResourceName(service.Name)) - template.Resources[taskExecutionRole] = &iam.Role{ - AssumeRolePolicyDocument: assumeRolePolicyDocument, - Policies: rolePolicies, - ManagedPolicyArns: []string{ - ECSTaskExecutionPolicy, - ECRReadOnlyPolicy, - }, - } template.Resources[taskDefinition] = definition var healthCheck *cloudmap.Service_HealthCheckConfig @@ -158,26 +105,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err // FIXME ECS only support HTTP(s) health checks, while Docker only support CMD } - serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", normalizeResourceName(service.Name)) - serviceRegistry := ecs.Service_ServiceRegistry{ - RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), - } - - template.Resources[serviceRegistration] = &cloudmap.Service{ - Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), - HealthCheckConfig: healthCheck, - Name: service.Name, - NamespaceId: cloudformation.Ref("CloudMap"), - DnsConfig: &cloudmap.Service_DnsConfig{ - DnsRecords: []cloudmap.Service_DnsRecord{ - { - TTL: 60, - Type: cloudmapapi.RecordTypeA, - }, - }, - RoutingPolicy: cloudmapapi.RoutingPolicyMultivalue, - }, - } + serviceRegistry := c.createServiceRegistry(service, template, healthCheck) serviceSecurityGroups := []string{} for net := range service.Networks { @@ -188,54 +116,10 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err serviceLB := []ecs.Service_LoadBalancer{} if len(service.Ports) > 0 { for _, port := range service.Ports { - - protocolType := strings.ToUpper(port.Protocol) - targetGroupName := fmt.Sprintf( - "%s%s%sTargetGroup", - normalizeResourceName(service.Name), - strings.ToUpper(port.Protocol), - string(port.Published), - ) - template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{ - Name: targetGroupName, - Port: int(port.Target), - Protocol: protocolType, - Tags: []tags.Tag{ - { - Key: ProjectTag, - Value: project.Name, - }, - }, - VpcId: cloudformation.Ref(ParameterVPCId), - TargetType: elbv2.TargetTypeEnumIp, - } - listenerName := fmt.Sprintf( - "%s%s%sListener", - normalizeResourceName(service.Name), - strings.ToUpper(port.Protocol), - string(port.Published), - ) - //add listener to dependsOn - //https://stackoverflow.com/questions/53971873/the-target-group-does-not-have-an-associated-load-balancer + protocol := strings.ToUpper(port.Protocol) + targetGroupName := c.createTargetGroup(project, service, port, template, protocol) + listenerName := c.createListener(service, port, template, targetGroupName, loadBalancer, protocol) dependsOn = append(dependsOn, listenerName) - template.Resources[listenerName] = &elasticloadbalancingv2.Listener{ - DefaultActions: []elasticloadbalancingv2.Listener_Action{ - { - ForwardConfig: &elasticloadbalancingv2.Listener_ForwardConfig{ - TargetGroups: []elasticloadbalancingv2.Listener_TargetGroupTuple{ - { - TargetGroupArn: cloudformation.Ref(targetGroupName), - }, - }, - }, - Type: elbv2.ActionTypeEnumForward, - }, - }, - LoadBalancerArn: cloudformation.Ref(loadBalancerName), - Protocol: protocolType, - Port: int(port.Published), - } - serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ ContainerName: service.Name, ContainerPort: int(port.Published), @@ -287,6 +171,156 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return template, nil } +func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template) string { + loadBalancerType := "network" + loadBalancerName := fmt.Sprintf( + "%s%sLB", + strings.Title(project.Name), + strings.ToUpper(loadBalancerType[0:1]), + ) + loadBalancer := &elasticloadbalancingv2.LoadBalancer{ + Name: loadBalancerName, + Scheme: "internet-facing", + Subnets: []string{ + cloudformation.Ref(ParameterSubnet1Id), + cloudformation.Ref(ParameterSubnet2Id), + }, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + }, + Type: loadBalancerType, + } + template.Resources[loadBalancerName] = loadBalancer + return loadBalancerName +} + +func (c client) createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerName string, protocol string) string { + listenerName := fmt.Sprintf( + "%s%s%sListener", + normalizeResourceName(service.Name), + strings.ToUpper(port.Protocol), + string(port.Published), + ) + //add listener to dependsOn + //https://stackoverflow.com/questions/53971873/the-target-group-does-not-have-an-associated-load-balancer + template.Resources[listenerName] = &elasticloadbalancingv2.Listener{ + DefaultActions: []elasticloadbalancingv2.Listener_Action{ + { + ForwardConfig: &elasticloadbalancingv2.Listener_ForwardConfig{ + TargetGroups: []elasticloadbalancingv2.Listener_TargetGroupTuple{ + { + TargetGroupArn: cloudformation.Ref(targetGroupName), + }, + }, + }, + Type: elbv2.ActionTypeEnumForward, + }, + }, + LoadBalancerArn: cloudformation.Ref(loadBalancerName), + Protocol: protocol, + Port: int(port.Published), + } + return listenerName +} + +func (c client) createTargetGroup(project *compose.Project, service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, protocol string) string { + targetGroupName := fmt.Sprintf( + "%s%s%sTargetGroup", + normalizeResourceName(service.Name), + strings.ToUpper(port.Protocol), + string(port.Published), + ) + template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{ + Name: targetGroupName, + Port: int(port.Target), + Protocol: protocol, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + }, + VpcId: cloudformation.Ref(ParameterVPCId), + TargetType: elbv2.TargetTypeEnumIp, + } + return targetGroupName +} + +func (c client) createServiceRegistry(service types.ServiceConfig, template *cloudformation.Template, healthCheck *cloudmap.Service_HealthCheckConfig) ecs.Service_ServiceRegistry { + serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", normalizeResourceName(service.Name)) + serviceRegistry := ecs.Service_ServiceRegistry{ + RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), + } + + template.Resources[serviceRegistration] = &cloudmap.Service{ + Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), + HealthCheckConfig: healthCheck, + Name: service.Name, + NamespaceId: cloudformation.Ref("CloudMap"), + DnsConfig: &cloudmap.Service_DnsConfig{ + DnsRecords: []cloudmap.Service_DnsRecord{ + { + TTL: 60, + Type: cloudmapapi.RecordTypeA, + }, + }, + RoutingPolicy: cloudmapapi.RoutingPolicyMultivalue, + }, + } + return serviceRegistry +} + +func (c client) createTaskExecutionRole(service types.ServiceConfig, err error, definition *ecs.TaskDefinition, template *cloudformation.Template) (string, error) { + taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", normalizeResourceName(service.Name)) + policy, err := c.getPolicy(definition) + if err != nil { + return taskExecutionRole, err + } + rolePolicies := []iam.Role_Policy{} + if policy != nil { + rolePolicies = append(rolePolicies, iam.Role_Policy{ + PolicyDocument: policy, + PolicyName: fmt.Sprintf("%sGrantAccessToSecrets", service.Name), + }) + + } + template.Resources[taskExecutionRole] = &iam.Role{ + AssumeRolePolicyDocument: assumeRolePolicyDocument, + Policies: rolePolicies, + ManagedPolicyArns: []string{ + ECSTaskExecutionPolicy, + ECRReadOnlyPolicy, + }, + } + return taskExecutionRole, nil +} + +func (c client) createCluster(project *compose.Project, template *cloudformation.Template) string { + template.Resources["Cluster"] = &ecs.Cluster{ + ClusterName: project.Name, + Tags: []tags.Tag{ + { + Key: ProjectTag, + Value: project.Name, + }, + }, + AWSCloudFormationCondition: "CreateCluster", + } + cluster := cloudformation.If("CreateCluster", cloudformation.Ref("Cluster"), cloudformation.Ref(ParameterClusterName)) + return cluster +} + +func (c client) createCloudMap(project *compose.Project, template *cloudformation.Template) { + template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ + Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), + Name: fmt.Sprintf("%s.local", project.Name), + Vpc: cloudformation.Ref(ParameterVPCId), + } +} + func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc string, template *cloudformation.Template) string { if sg, ok := net.Extras[ExtensionSecurityGroup]; ok { logrus.Debugf("Security Group for network %q set by user to %q", net.Name, sg) From fbb5bdac6ee132ce6565a7eefd77894f65d3a6c5 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 4 Jun 2020 16:20:21 +0200 Subject: [PATCH 102/205] Fix resource naming Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 22 ++++++++-------------- ecs/pkg/compose/normalize.go | 7 +++++++ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 0e6966bb..35d2e123 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -172,15 +172,10 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template) string { - loadBalancerType := "network" - loadBalancerName := fmt.Sprintf( - "%s%sLB", - strings.Title(project.Name), - strings.ToUpper(loadBalancerType[0:1]), - ) - loadBalancer := &elasticloadbalancingv2.LoadBalancer{ + loadBalancerName := fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name)) + template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ Name: loadBalancerName, - Scheme: "internet-facing", + Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, Subnets: []string{ cloudformation.Ref(ParameterSubnet1Id), cloudformation.Ref(ParameterSubnet2Id), @@ -191,18 +186,17 @@ func (c client) createLoadBalancer(project *compose.Project, template *cloudform Value: project.Name, }, }, - Type: loadBalancerType, + Type: elbv2.LoadBalancerTypeEnumNetwork, } - template.Resources[loadBalancerName] = loadBalancer return loadBalancerName } func (c client) createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerName string, protocol string) string { listenerName := fmt.Sprintf( - "%s%s%sListener", + "%s%s%dListener", normalizeResourceName(service.Name), strings.ToUpper(port.Protocol), - string(port.Published), + port.Published, ) //add listener to dependsOn //https://stackoverflow.com/questions/53971873/the-target-group-does-not-have-an-associated-load-balancer @@ -228,10 +222,10 @@ func (c client) createListener(service types.ServiceConfig, port types.ServicePo func (c client) createTargetGroup(project *compose.Project, service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, protocol string) string { targetGroupName := fmt.Sprintf( - "%s%s%sTargetGroup", + "%s%s%dTargetGroup", normalizeResourceName(service.Name), strings.ToUpper(port.Protocol), - string(port.Published), + port.Published, ) template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{ Name: targetGroupName, diff --git a/ecs/pkg/compose/normalize.go b/ecs/pkg/compose/normalize.go index 861d146f..3e4809d1 100644 --- a/ecs/pkg/compose/normalize.go +++ b/ecs/pkg/compose/normalize.go @@ -23,6 +23,13 @@ func Normalize(model *types.Config) error { s.Networks = map[string]*types.ServiceNetworkConfig{"default": nil} } + for i, p := range s.Ports { + if p.Published == 0 { + p.Published = p.Target + s.Ports[i] = p + } + } + if s.LogDriver != "" { logrus.Warn("`log_driver` is deprecated. Use the `logging` attribute") if s.Logging == nil { From 3194cc9b1617224e3d486cda02a27d2682c59951 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 4 Jun 2020 16:28:11 +0200 Subject: [PATCH 103/205] allow user defined LB Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 41 +++++++++++++++++++++++--------- ecs/pkg/amazon/sdk.go | 22 +++++++++++++++++ ecs/pkg/amazon/up.go | 33 +++++++++++++++++++++---- ecs/pkg/amazon/x.go | 1 + 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 35d2e123..68f6b344 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -25,10 +25,11 @@ import ( ) const ( - ParameterClusterName = "ParameterClusterName" - ParameterVPCId = "ParameterVPCId" - ParameterSubnet1Id = "ParameterSubnet1Id" - ParameterSubnet2Id = "ParameterSubnet2Id" + ParameterClusterName = "ParameterClusterName" + ParameterVPCId = "ParameterVPCId" + ParameterSubnet1Id = "ParameterSubnet1Id" + ParameterSubnet2Id = "ParameterSubnet2Id" + ParameterLoadBalancerARN = "ParameterLoadBalancerARN" ) // Convert a compose project into a CloudFormation template @@ -66,6 +67,11 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err Description: "SubnetId, for Availability Zone 2 in the region in your VPC", } + template.Parameters[ParameterLoadBalancerARN] = cloudformation.Parameter{ + Type: "String", + Description: "Name of the LoadBalancer to connect to (optional)", + } + // Create Cluster is `ParameterClusterName` parameter is not set template.Conditions["CreateCluster"] = cloudformation.Equals("", cloudformation.Ref(ParameterClusterName)) @@ -172,10 +178,19 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template) string { - loadBalancerName := fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name)) - template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ + + loadBalancerType := "network" + loadBalancerName := fmt.Sprintf( + "%s%sLB", + strings.Title(project.Name), + strings.ToUpper(loadBalancerType[0:1]), + ) + // Create LoadBalancer if `ParameterLoadBalancerName` is not set + template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) + + loadBalancer := &elasticloadbalancingv2.LoadBalancer{ Name: loadBalancerName, - Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, + Scheme: "internet-facing", Subnets: []string{ cloudformation.Ref(ParameterSubnet1Id), cloudformation.Ref(ParameterSubnet2Id), @@ -186,12 +201,16 @@ func (c client) createLoadBalancer(project *compose.Project, template *cloudform Value: project.Name, }, }, - Type: elbv2.LoadBalancerTypeEnumNetwork, + Type: loadBalancerType, + AWSCloudFormationCondition: "CreateLoadBalancer", } - return loadBalancerName + template.Resources[loadBalancerName] = loadBalancer + loadBalancerRef := cloudformation.If("CreateLoadBalancer", cloudformation.Ref(loadBalancerName), cloudformation.Ref(ParameterLoadBalancerARN)) + + return loadBalancerRef } -func (c client) createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerName string, protocol string) string { +func (c client) createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerARN string, protocol string) string { listenerName := fmt.Sprintf( "%s%s%dListener", normalizeResourceName(service.Name), @@ -213,7 +232,7 @@ func (c client) createListener(service types.ServiceConfig, port types.ServicePo Type: elbv2.ActionTypeEnumForward, }, }, - LoadBalancerArn: cloudformation.Ref(loadBalancerName), + LoadBalancerArn: loadBalancerARN, Protocol: protocol, Port: int(port.Published), } diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk.go index 6bc85381..e074fb11 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk.go @@ -400,3 +400,25 @@ func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string } return publicIPs, nil } + +func (s sdk) LoadBalancerExists(ctx context.Context, name string) (bool, error) { + logrus.Debug("Check if cluster was already created: ", name) + lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{ + Names: []*string{aws.String(name)}, + }) + if err != nil { + return false, err + } + return len(lbs.LoadBalancers) > 0, nil +} + +func (s sdk) GetLoadBalancerARN(ctx context.Context, name string) (string, error) { + logrus.Debug("Check if cluster was already created: ", name) + lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{ + Names: []*string{aws.String(name)}, + }) + if err != nil { + return "", err + } + return *lbs.LoadBalancers[0].LoadBalancerArn, nil +} diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go index 21500c38..3c947724 100644 --- a/ecs/pkg/amazon/up.go +++ b/ecs/pkg/amazon/up.go @@ -42,11 +42,17 @@ func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error return err } + lb, err := c.GetLoadBalancer(ctx, project) + if err != nil { + return err + } + parameters := map[string]string{ - ParameterClusterName: c.Cluster, - ParameterVPCId: vpc, - ParameterSubnet1Id: subNets[0], - ParameterSubnet2Id: subNets[1], + ParameterClusterName: c.Cluster, + ParameterVPCId: vpc, + ParameterSubnet1Id: subNets[0], + ParameterSubnet2Id: subNets[1], + ParameterLoadBalancerARN: lb, } err = c.api.CreateStack(ctx, project.Name, template, parameters) @@ -77,6 +83,22 @@ func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, e return defaultVPC, nil } +func (c client) GetLoadBalancer(ctx context.Context, project *compose.Project) (string, error) { + //check compose file for custom VPC selected + if lb, ok := project.Extras[ExtensionLB]; ok { + lbName := lb.(string) + ok, err := c.api.LoadBalancerExists(ctx, lbName) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("Load Balancer does not exist: %s", lb) + } + return c.api.GetLoadBalancerARN(ctx, lbName) + } + return "", nil +} + type upAPI interface { waitAPI GetDefaultVPC(ctx context.Context) (string, error) @@ -86,4 +108,7 @@ type upAPI interface { ClusterExists(ctx context.Context, name string) (bool, error) StackExists(ctx context.Context, name string) (bool, error) CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error + + LoadBalancerExists(ctx context.Context, name string) (bool, error) + GetLoadBalancerARN(ctx context.Context, name string) (string, error) } diff --git a/ecs/pkg/amazon/x.go b/ecs/pkg/amazon/x.go index 0b022bac..b16c3d25 100644 --- a/ecs/pkg/amazon/x.go +++ b/ecs/pkg/amazon/x.go @@ -4,4 +4,5 @@ const ( ExtensionSecurityGroup = "x-aws-securitygroup" ExtensionVPC = "x-aws-vpc" ExtensionPullCredentials = "x-aws-pull_credentials" + ExtensionLB = "x-aws-lb" ) From 2ea694a1c5b7331e8c3820cae9ad9c876ca5e5a6 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 4 Jun 2020 17:09:41 +0200 Subject: [PATCH 104/205] update test data Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/api_mock.go | 27 ++++++++++++++++++- .../simple-cloudformation-conversion.golden | 13 +++++++++ ...formation-with-overrides-conversion.golden | 13 +++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/api_mock.go b/ecs/pkg/amazon/api_mock.go index 40e9f687..cbf2b6d1 100644 --- a/ecs/pkg/amazon/api_mock.go +++ b/ecs/pkg/amazon/api_mock.go @@ -6,11 +6,12 @@ package amazon import ( context "context" + reflect "reflect" + cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" docker "github.com/docker/ecs-plugin/pkg/docker" gomock "github.com/golang/mock/gomock" - reflect "reflect" ) // MockAPI is a mock of API interface @@ -324,3 +325,27 @@ func (mr *MockAPIMockRecorder) WaitStackComplete(arg0, arg1, arg2 interface{}) * mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitStackComplete", reflect.TypeOf((*MockAPI)(nil).WaitStackComplete), arg0, arg1, arg2) } + +// LoadBalancerExists mocks base method +func (m *MockAPI) LoadBalancerExists(arg0 context.Context, arg1 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerExists", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadBalancerExists indicates an expected call of VpcExists +func (mr *MockAPIMockRecorder) LoadBalancerExists(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerExists", reflect.TypeOf((*MockAPI)(nil).LoadBalancerExists), arg0, arg1) +} + +// GetLoadBalancerARN mocks base method +func (m *MockAPI) GetLoadBalancerARN(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoadBalancerARN", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 258fa960..15006580 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -8,6 +8,14 @@ "Ref": "ParameterClusterName" } ] + }, + "CreateLoadBalancer": { + "Fn::Equals": [ + "", + { + "Ref": "ParameterLoadBalancerARN" + } + ] } }, "Parameters": { @@ -15,6 +23,10 @@ "Description": "Name of the ECS cluster to deploy to (optional)", "Type": "String" }, + "ParameterLoadBalancerARN": { + "Description": "Name of the LoadBalancer to connect to (optional)", + "Type": "String" + }, "ParameterSubnet1Id": { "Description": "SubnetId, for Availability Zone 1 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" @@ -247,6 +259,7 @@ "Type": "AWS::EC2::SecurityGroupIngress" }, "TestSimpleConvertNLB": { + "Condition": "CreateLoadBalancer", "Properties": { "Name": "TestSimpleConvertNLB", "Scheme": "internet-facing", diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 991f65b6..0324c289 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -8,6 +8,14 @@ "Ref": "ParameterClusterName" } ] + }, + "CreateLoadBalancer": { + "Fn::Equals": [ + "", + { + "Ref": "ParameterLoadBalancerARN" + } + ] } }, "Parameters": { @@ -15,6 +23,10 @@ "Description": "Name of the ECS cluster to deploy to (optional)", "Type": "String" }, + "ParameterLoadBalancerARN": { + "Description": "Name of the LoadBalancer to connect to (optional)", + "Type": "String" + }, "ParameterSubnet1Id": { "Description": "SubnetId, for Availability Zone 1 in the region in your VPC", "Type": "AWS::EC2::Subnet::Id" @@ -247,6 +259,7 @@ "Type": "AWS::EC2::SecurityGroupIngress" }, "TestSimpleWithOverridesNLB": { + "Condition": "CreateLoadBalancer", "Properties": { "Name": "TestSimpleWithOverridesNLB", "Scheme": "internet-facing", From 02cc644c5af2e56846ccc5ccc67395a019f4128b Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 4 Jun 2020 18:04:16 +0200 Subject: [PATCH 105/205] fix test data Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 17 +++++------------ .../simple-cloudformation-conversion.golden | 4 ++-- ...udformation-with-overrides-conversion.golden | 4 ++-- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 68f6b344..24b5786d 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -89,7 +89,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err // Private DNS namespace will allow DNS name for the services to be ..local c.createCloudMap(project, template) - loadBalancer := c.createLoadBalancer(project, template) + loadBalancer := c.createLoadBalancer(project, template, "network") for _, service := range project.Services { definition, err := Convert(project, service) @@ -177,20 +177,14 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return template, nil } -func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template) string { - - loadBalancerType := "network" - loadBalancerName := fmt.Sprintf( - "%s%sLB", - strings.Title(project.Name), - strings.ToUpper(loadBalancerType[0:1]), - ) +func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template, loadBalancerType string) string { + loadBalancerName := fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name)) // Create LoadBalancer if `ParameterLoadBalancerName` is not set template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) - loadBalancer := &elasticloadbalancingv2.LoadBalancer{ + template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ Name: loadBalancerName, - Scheme: "internet-facing", + Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, Subnets: []string{ cloudformation.Ref(ParameterSubnet1Id), cloudformation.Ref(ParameterSubnet2Id), @@ -204,7 +198,6 @@ func (c client) createLoadBalancer(project *compose.Project, template *cloudform Type: loadBalancerType, AWSCloudFormationCondition: "CreateLoadBalancer", } - template.Resources[loadBalancerName] = loadBalancer loadBalancerRef := cloudformation.If("CreateLoadBalancer", cloudformation.Ref(loadBalancerName), cloudformation.Ref(ParameterLoadBalancerARN)) return loadBalancerRef diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 15006580..15222e57 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -258,10 +258,10 @@ }, "Type": "AWS::EC2::SecurityGroupIngress" }, - "TestSimpleConvertNLB": { + "TestSimpleConvertLoadBalancer": { "Condition": "CreateLoadBalancer", "Properties": { - "Name": "TestSimpleConvertNLB", + "Name": "TestSimpleConvertLoadBalancer", "Scheme": "internet-facing", "Subnets": [ { diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 0324c289..5cc9cb98 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -258,10 +258,10 @@ }, "Type": "AWS::EC2::SecurityGroupIngress" }, - "TestSimpleWithOverridesNLB": { + "TestSimpleWithOverridesLoadBalancer": { "Condition": "CreateLoadBalancer", "Properties": { - "Name": "TestSimpleWithOverridesNLB", + "Name": "TestSimpleWithOverridesLoadBalancer", "Scheme": "internet-facing", "Subnets": [ { From 7337c7520fad4f6ecf752b60567aca9870c3ca9c Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 4 Jun 2020 18:23:31 +0200 Subject: [PATCH 106/205] rename LB field in the compose file Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/x.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/x.go b/ecs/pkg/amazon/x.go index b16c3d25..315c50c4 100644 --- a/ecs/pkg/amazon/x.go +++ b/ecs/pkg/amazon/x.go @@ -4,5 +4,5 @@ const ( ExtensionSecurityGroup = "x-aws-securitygroup" ExtensionVPC = "x-aws-vpc" ExtensionPullCredentials = "x-aws-pull_credentials" - ExtensionLB = "x-aws-lb" + ExtensionLB = "x-aws-loadbalancer" ) From dad36e09f9b1f04a7ad5106cdf8bbee65cc48c93 Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 5 Jun 2020 10:32:30 +0200 Subject: [PATCH 107/205] set ALB and security groups for http(s) protocol Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 54 +++++++++++++++++-- .../simple-cloudformation-conversion.golden | 2 +- ...formation-with-overrides-conversion.golden | 2 +- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 24b5786d..8c3e5e41 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -89,7 +89,9 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err // Private DNS namespace will allow DNS name for the services to be ..local c.createCloudMap(project, template) - loadBalancer := c.createLoadBalancer(project, template, "network") + + loadBalancerType, albSecurityGroups := c.getLoadBalancerType(project, networks) + loadBalancer := c.createLoadBalancer(project, template, loadBalancerType, albSecurityGroups) for _, service := range project.Services { definition, err := Convert(project, service) @@ -123,6 +125,12 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err if len(service.Ports) > 0 { for _, port := range service.Ports { protocol := strings.ToUpper(port.Protocol) + if loadBalancerType == elbv2.LoadBalancerTypeEnumApplication { + protocol = elbv2.ProtocolEnumHttps + if port.Published == 80 { + protocol = elbv2.ProtocolEnumHttp + } + } targetGroupName := c.createTargetGroup(project, service, port, template, protocol) listenerName := c.createListener(service, port, template, targetGroupName, loadBalancer, protocol) dependsOn = append(dependsOn, listenerName) @@ -177,14 +185,40 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return template, nil } -func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template, loadBalancerType string) string { +func (c client) getLoadBalancerType(project *compose.Project, networks map[string]string) (string, []string) { + // check what type of load balancer to create, we asssume by default application type + loadBalancerType := elbv2.LoadBalancerTypeEnumApplication + albSecurityGroups := []string{} + + for _, service := range project.Services { + if len(service.Ports) == 0 { + continue + } + for _, port := range service.Ports { + if port.Published != 80 && port.Published != 443 { + return elbv2.LoadBalancerTypeEnumNetwork, []string{} + } + } + + serviceSecurityGroups := []string{} + for net := range service.Networks { + serviceSecurityGroups = append(serviceSecurityGroups, networks[net]) + } + albSecurityGroups = append(albSecurityGroups, serviceSecurityGroups...) + albSecurityGroups = uniqueStrings(albSecurityGroups) + } + return loadBalancerType, albSecurityGroups +} + +func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template, loadBalancerType string, securityGroups []string) string { loadBalancerName := fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name)) // Create LoadBalancer if `ParameterLoadBalancerName` is not set template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ - Name: loadBalancerName, - Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, + Name: loadBalancerName, + Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, + SecurityGroups: securityGroups, Subnets: []string{ cloudformation.Ref(ParameterSubnet1Id), cloudformation.Ref(ParameterSubnet2Id), @@ -417,3 +451,15 @@ func (c client) getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) } return nil, nil } + +func uniqueStrings(items []string) []string { + keys := make(map[string]bool) + unique := []string{} + for _, item := range items { + if _, val := keys[item]; !val { + keys[item] = true + unique = append(unique, item) + } + } + return unique +} diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 15222e57..3b9e55c4 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -277,7 +277,7 @@ "Value": "TestSimpleConvert" } ], - "Type": "network" + "Type": "application" }, "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" } diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 5cc9cb98..80c9b2aa 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -277,7 +277,7 @@ "Value": "TestSimpleWithOverrides" } ], - "Type": "network" + "Type": "application" }, "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" } From c04950cdac7755a1d1f3dd7bb348586f81de08e6 Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 5 Jun 2020 11:34:02 +0200 Subject: [PATCH 108/205] improve lb security groups parsing Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 8c3e5e41..955a8c2b 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -186,28 +186,22 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } func (c client) getLoadBalancerType(project *compose.Project, networks map[string]string) (string, []string) { - // check what type of load balancer to create, we asssume by default application type - loadBalancerType := elbv2.LoadBalancerTypeEnumApplication - albSecurityGroups := []string{} - for _, service := range project.Services { - if len(service.Ports) == 0 { - continue - } for _, port := range service.Ports { if port.Published != 80 && port.Published != 443 { return elbv2.LoadBalancerTypeEnumNetwork, []string{} } } - - serviceSecurityGroups := []string{} - for net := range service.Networks { - serviceSecurityGroups = append(serviceSecurityGroups, networks[net]) - } - albSecurityGroups = append(albSecurityGroups, serviceSecurityGroups...) - albSecurityGroups = uniqueStrings(albSecurityGroups) } - return loadBalancerType, albSecurityGroups + + albSecurityGroups := []string{} + for _, network := range project.Networks { + if !network.Internal { + albSecurityGroups = append(albSecurityGroups, networks[network.Name]) + } + } + albSecurityGroups = uniqueStrings(albSecurityGroups) + return elbv2.LoadBalancerTypeEnumApplication, albSecurityGroups } func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template, loadBalancerType string, securityGroups []string) string { From 45dc8eda80dd3e773228b9c20eb13b31338a791e Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 5 Jun 2020 11:38:27 +0200 Subject: [PATCH 109/205] update test data Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- .../testdata/simple/simple-cloudformation-conversion.golden | 5 +++++ .../simple-cloudformation-with-overrides-conversion.golden | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 3b9e55c4..7c7af678 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -263,6 +263,11 @@ "Properties": { "Name": "TestSimpleConvertLoadBalancer", "Scheme": "internet-facing", + "SecurityGroups": [ + { + "Ref": "TestSimpleConvertDefaultNetwork" + } + ], "Subnets": [ { "Ref": "ParameterSubnet1Id" diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 80c9b2aa..e9535e90 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -263,6 +263,11 @@ "Properties": { "Name": "TestSimpleWithOverridesLoadBalancer", "Scheme": "internet-facing", + "SecurityGroups": [ + { + "Ref": "TestSimpleWithOverridesDefaultNetwork" + } + ], "Subnets": [ { "Ref": "ParameterSubnet1Id" From c0f1a8bf18bab3dda418728c2fc4b345889a852a Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 5 Jun 2020 15:14:50 +0200 Subject: [PATCH 110/205] create different methods to get lb type and security groups Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 44 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 955a8c2b..c9503491 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -90,8 +90,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err // Private DNS namespace will allow DNS name for the services to be ..local c.createCloudMap(project, template) - loadBalancerType, albSecurityGroups := c.getLoadBalancerType(project, networks) - loadBalancer := c.createLoadBalancer(project, template, loadBalancerType, albSecurityGroups) + loadBalancerARN := c.createLoadBalancer(project, template) for _, service := range project.Services { definition, err := Convert(project, service) @@ -125,14 +124,14 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err if len(service.Ports) > 0 { for _, port := range service.Ports { protocol := strings.ToUpper(port.Protocol) - if loadBalancerType == elbv2.LoadBalancerTypeEnumApplication { + if c.getLoadBalancerType(project) == elbv2.LoadBalancerTypeEnumApplication { protocol = elbv2.ProtocolEnumHttps if port.Published == 80 { protocol = elbv2.ProtocolEnumHttp } } targetGroupName := c.createTargetGroup(project, service, port, template, protocol) - listenerName := c.createListener(service, port, template, targetGroupName, loadBalancer, protocol) + listenerName := c.createListener(service, port, template, targetGroupName, loadBalancerARN, protocol) dependsOn = append(dependsOn, listenerName) serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ ContainerName: service.Name, @@ -185,30 +184,39 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return template, nil } -func (c client) getLoadBalancerType(project *compose.Project, networks map[string]string) (string, []string) { +func (c client) getLoadBalancerType(project *compose.Project) string { for _, service := range project.Services { for _, port := range service.Ports { if port.Published != 80 && port.Published != 443 { - return elbv2.LoadBalancerTypeEnumNetwork, []string{} + return elbv2.LoadBalancerTypeEnumNetwork } } } - - albSecurityGroups := []string{} - for _, network := range project.Networks { - if !network.Internal { - albSecurityGroups = append(albSecurityGroups, networks[network.Name]) - } - } - albSecurityGroups = uniqueStrings(albSecurityGroups) - return elbv2.LoadBalancerTypeEnumApplication, albSecurityGroups + return elbv2.LoadBalancerTypeEnumApplication } -func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template, loadBalancerType string, securityGroups []string) string { +func (c client) getLoadBalancerSecurityGroups(project *compose.Project, template *cloudformation.Template) []string { + securityGroups := []string{} + for _, network := range project.Networks { + if !network.Internal { + net := convertNetwork(project, network, cloudformation.Ref(ParameterVPCId), template) + securityGroups = append(securityGroups, net) + } + } + return uniqueStrings(securityGroups) +} + +func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template) string { loadBalancerName := fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name)) // Create LoadBalancer if `ParameterLoadBalancerName` is not set template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) + loadBalancerType := c.getLoadBalancerType(project) + securityGroups := []string{} + if loadBalancerType == elbv2.LoadBalancerTypeEnumApplication { + securityGroups = c.getLoadBalancerSecurityGroups(project, template) + } + template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ Name: loadBalancerName, Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, @@ -226,9 +234,7 @@ func (c client) createLoadBalancer(project *compose.Project, template *cloudform Type: loadBalancerType, AWSCloudFormationCondition: "CreateLoadBalancer", } - loadBalancerRef := cloudformation.If("CreateLoadBalancer", cloudformation.Ref(loadBalancerName), cloudformation.Ref(ParameterLoadBalancerARN)) - - return loadBalancerRef + return cloudformation.If("CreateLoadBalancer", cloudformation.Ref(loadBalancerName), cloudformation.Ref(ParameterLoadBalancerARN)) } func (c client) createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerARN string, protocol string) string { From e88b11bc2693d6491176c34d662ff14a1828f1c3 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 8 Jun 2020 09:16:58 +0200 Subject: [PATCH 111/205] Introduce test to check CloudFormation conversion Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation_test.go | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/ecs/pkg/amazon/cloudformation_test.go b/ecs/pkg/amazon/cloudformation_test.go index 456efbd0..84b7c9bc 100644 --- a/ecs/pkg/amazon/cloudformation_test.go +++ b/ecs/pkg/amazon/cloudformation_test.go @@ -4,6 +4,12 @@ import ( "fmt" "testing" + "github.com/awslabs/goformation/v4/cloudformation/ec2" + + "github.com/awslabs/goformation/v4/cloudformation" + "github.com/compose-spec/compose-go/loader" + "github.com/compose-spec/compose-go/types" + "gotest.tools/assert" "github.com/docker/ecs-plugin/pkg/compose" @@ -24,6 +30,27 @@ func TestSimpleWithOverrides(t *testing.T) { golden.Assert(t, result, expected) } +func TestMapNetworksToSecurityGroups(t *testing.T) { + template := convertYaml(t, ` +version: "3" +services: + test: + image: hello_world +networks: + front-tier: + name: public + back-tier: + internal: true +`) + assert.Check(t, template.Resources["TestPublicNetwork"] != nil) + assert.Check(t, template.Resources["TestBacktierNetwork"] != nil) + assert.Check(t, template.Resources["TestBacktierNetworkIngress"] != nil) + ingress := template.Resources["TestPublicNetworkIngress"].(*ec2.SecurityGroupIngress) + assert.Check(t, ingress != nil) + assert.Check(t, ingress.SourceSecurityGroupId == cloudformation.Ref("TestPublicNetwork")) + +} + func convertResultAsString(t *testing.T, project *compose.Project, clusterName string) string { client, err := NewClient("", clusterName, "") assert.NilError(t, err) @@ -43,3 +70,22 @@ func load(t *testing.T, paths ...string) *compose.Project { assert.NilError(t, err) return project } + +func convertYaml(t *testing.T, yaml string) *cloudformation.Template { + dict, err := loader.ParseYAML([]byte(yaml)) + assert.NilError(t, err) + model, err := loader.Load(types.ConfigDetails{ + ConfigFiles: []types.ConfigFile{ + {Config: dict}, + }, + }) + assert.NilError(t, err) + err = compose.Normalize(model) + assert.NilError(t, err) + template, err := client{}.Convert(&compose.Project{ + Config: *model, + Name: "test", + }) + assert.NilError(t, err) + return template +} From 1d11e847fb234912c7b609cecccbf22036aa17d6 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 8 Jun 2020 11:28:15 +0200 Subject: [PATCH 112/205] Test we create the expected policy document for pull_credentials Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 1 - ecs/pkg/amazon/cloudformation_test.go | 28 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index c9503491..6afa06c3 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -426,7 +426,6 @@ func normalizeResourceName(s string) string { } func (c client) getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { - arns := []string{} for _, container := range taskDef.ContainerDefinitions { if container.RepositoryCredentials != nil { diff --git a/ecs/pkg/amazon/cloudformation_test.go b/ecs/pkg/amazon/cloudformation_test.go index 84b7c9bc..e23079d5 100644 --- a/ecs/pkg/amazon/cloudformation_test.go +++ b/ecs/pkg/amazon/cloudformation_test.go @@ -4,15 +4,13 @@ import ( "fmt" "testing" - "github.com/awslabs/goformation/v4/cloudformation/ec2" - "github.com/awslabs/goformation/v4/cloudformation" + "github.com/awslabs/goformation/v4/cloudformation/ec2" + "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" - - "gotest.tools/assert" - "github.com/docker/ecs-plugin/pkg/compose" + "gotest.tools/assert" "gotest.tools/v3/golden" ) @@ -30,6 +28,26 @@ func TestSimpleWithOverrides(t *testing.T) { golden.Assert(t, result, expected) } +func TestRolePolicy(t *testing.T) { + template := convertYaml(t, ` +version: "3" +services: + foo: + image: hello_world + x-aws-pull_credentials: "secret" +`) + role := template.Resources["FooTaskExecutionRole"].(*iam.Role) + assert.Check(t, role != nil) + assert.Check(t, role.ManagedPolicyArns[0] == ECSTaskExecutionPolicy) + assert.Check(t, role.ManagedPolicyArns[1] == ECRReadOnlyPolicy) + // We expect an extra policy has been created for x-aws-pull_credentials + assert.Check(t, len(role.Policies) == 1) + policy := role.Policies[0].PolicyDocument.(*PolicyDocument) + expected := []string{"secretsmanager:GetSecretValue", "ssm:GetParameters", "kms:Decrypt"} + assert.DeepEqual(t, expected, policy.Statement[0].Action) + assert.DeepEqual(t, []string{"secret"}, policy.Statement[0].Resource) +} + func TestMapNetworksToSecurityGroups(t *testing.T) { template := convertYaml(t, ` version: "3" From 2c190f11f71696701dd98cd2b9e9f146069e87d9 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 8 Jun 2020 15:34:33 +0200 Subject: [PATCH 113/205] LB Type tests Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation_test.go | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ecs/pkg/amazon/cloudformation_test.go b/ecs/pkg/amazon/cloudformation_test.go index e23079d5..0c6ce62e 100644 --- a/ecs/pkg/amazon/cloudformation_test.go +++ b/ecs/pkg/amazon/cloudformation_test.go @@ -4,9 +4,13 @@ import ( "fmt" "testing" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/iam" + + "github.com/awslabs/goformation/v4/cloudformation" + "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" @@ -69,6 +73,36 @@ networks: } +func TestLoadBalancerTypeApplication(t *testing.T) { + template := convertYaml(t, ` +version: "3" +services: + test: + image: nginx + ports: + - 80:80 +`) + lb := template.Resources["TestLoadBalancer"].(*elasticloadbalancingv2.LoadBalancer) + assert.Check(t, lb != nil) + assert.Check(t, lb.Type == elbv2.LoadBalancerTypeEnumApplication) + assert.Check(t, len(lb.SecurityGroups) > 0) +} + +func TestLoadBalancerTypeNetwork(t *testing.T) { + template := convertYaml(t, ` +version: "3" +services: + test: + image: nginx + ports: + - 80:80 + - 88:88 +`) + lb := template.Resources["TestLoadBalancer"].(*elasticloadbalancingv2.LoadBalancer) + assert.Check(t, lb != nil) + assert.Check(t, lb.Type == elbv2.LoadBalancerTypeEnumNetwork) +} + func convertResultAsString(t *testing.T, project *compose.Project, clusterName string) string { client, err := NewClient("", clusterName, "") assert.NilError(t, err) From d597e55f229cf02a55c45026bf2229e41fbea374 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 8 Jun 2020 19:00:30 +0200 Subject: [PATCH 114/205] fix rebase Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ecs/pkg/amazon/cloudformation_test.go b/ecs/pkg/amazon/cloudformation_test.go index 0c6ce62e..84e12e76 100644 --- a/ecs/pkg/amazon/cloudformation_test.go +++ b/ecs/pkg/amazon/cloudformation_test.go @@ -9,7 +9,6 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/iam" - "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" From 5f628cd0e5ad8fe4cc460eee909ca8e471397670 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Mon, 8 Jun 2020 17:41:09 +0200 Subject: [PATCH 115/205] Refactor build process to build in containers Signed-off-by: Ulysses Souza Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 32 ++++++++++++++++++++++++++++++++ ecs/Makefile | 37 ++++++++++++++++++++++++++++++------- ecs/builder.Makefile | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 ecs/Dockerfile create mode 100644 ecs/builder.Makefile diff --git a/ecs/Dockerfile b/ecs/Dockerfile new file mode 100644 index 00000000..db374fb8 --- /dev/null +++ b/ecs/Dockerfile @@ -0,0 +1,32 @@ +# syntax = docker/dockerfile:experimental +ARG GO_VERSION=1.14.2 + +FROM golang:${GO_VERSION} AS base +ARG TARGET_OS=unknown +ARG TARGET_ARCH=unknown +ARG PWD=/ecs-plugin +ENV GO111MODULE=on + +WORKDIR ${PWD} +ADD go.* ${PWD} +RUN go mod download +ADD . ${PWD} + +FROM base AS make-plugin +RUN --mount=type=cache,target=/root/.cache/go-build \ + GOOS=${TARGET_OS} \ + GOARCH=${TARGET_ARCH} \ + make -f builder.Makefile build + +FROM base AS make-cross +RUN --mount=type=cache,target=/root/.cache/go-build \ + make -f builder.Makefile cross + +FROM scratch AS build +COPY --from=make-plugin /ecs-plugin/dist/* . + +FROM scratch AS cross +COPY --from=make-cross /ecs-plugin/dist/* . + +FROM base as test +RUN make -f builder.Makefile test diff --git a/ecs/Makefile b/ecs/Makefile index 0fd6b65a..c5fe2875 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -1,11 +1,27 @@ -clean: - rm -rf dist/ +GOOS ?= $(shell go env GOOS) +GOARCH ?= $(shell go env GOARCH) +PWD = $(shell pwd) -build: - go build -v -o dist/docker-ecs cmd/main/main.go +export DOCKER_BUILDKIT=1 + +.DEFAULT_GOAL := build + +build: ## Build for the current + @docker build . \ + --output type=local,dest=./dist \ + --build-arg TARGET_OS=${GOOS} \ + --build-arg TARGET_ARCH=${GOARCH} \ + --target build + +cross: ## Cross build for linux, macos and windows + @docker build . \ + --output type=local,dest=./dist \ + --target cross test: build ## Run tests - go test ./... -v + @docker build . \ + --output type=local,dest=./dist \ + --target test e2e: build ## Run tests go test ./... -v -tags=e2e @@ -15,6 +31,13 @@ dev: build ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" lint: ## Verify Go files - golangci-lint run --config ./golangci.yaml ./... + @docker run --rm -t \ + -v $(PWD):/app \ + -w /app \ + golangci/golangci-lint:v1.27-alpine \ + golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... -.PHONY: clean build test dev lint e2e +clean: + rm -rf dist/ + +.PHONY: clean build test dev lint e2e cross diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile new file mode 100644 index 00000000..0fd8e30b --- /dev/null +++ b/ecs/builder.Makefile @@ -0,0 +1,39 @@ +GOOS ?= $(shell go env GOOS) +GOARCH ?= $(shell go env GOARCH) + +PROTOS=$(shell find . -name \*.proto) + +EXTENSION := +ifeq ($(GOOS),windows) + EXTENSION := .exe +endif + +STATIC_FLAGS= CGO_ENABLED=0 +LDFLAGS := "-s -w" +GO_BUILD = $(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) + +BINARY=dist/docker +BINARY_WITH_EXTENSION=$(BINARY)$(EXTENSION) + +export DOCKER_BUILDKIT=1 + +all: build + +clean: + rm -rf dist/ + +build: + $(GO_BUILD) -v -o $(BINARY_WITH_EXTENSION) cmd/main/main.go + +cross: + @GOOS=linux GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-linux-amd64 cmd/main/main.go + @GOOS=darwin GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-darwin-amd64 cmd/main/main.go + @GOOS=windows GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-windows-amd64.exe cmd/main/main.go + +test: build ## Run tests + @go test ./... -v + +lint: ## Verify Go files + golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... + +.PHONY: clean build test dev lint e2e From f192904d4210e66db1f694fab6789e90eefdea4a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 9 Jun 2020 15:21:34 +0200 Subject: [PATCH 116/205] fix Makefile to produce docker-ecs binary Signed-off-by: Nicolas De Loof --- ecs/builder.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile index 0fd8e30b..81145e02 100644 --- a/ecs/builder.Makefile +++ b/ecs/builder.Makefile @@ -12,7 +12,7 @@ STATIC_FLAGS= CGO_ENABLED=0 LDFLAGS := "-s -w" GO_BUILD = $(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) -BINARY=dist/docker +BINARY=dist/docker-ecs BINARY_WITH_EXTENSION=$(BINARY)$(EXTENSION) export DOCKER_BUILDKIT=1 From 2ab64ea10ef534fa623e74b9f7edd1223ac46512 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Tue, 9 Jun 2020 13:51:46 +0200 Subject: [PATCH 117/205] docs: Add Linux install instructions Signed-off-by: Christopher Crone Signed-off-by: Nicolas De Loof --- ecs/docs/get-started-linux.md | 84 +++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 ecs/docs/get-started-linux.md diff --git a/ecs/docs/get-started-linux.md b/ecs/docs/get-started-linux.md new file mode 100644 index 00000000..2b1a129b --- /dev/null +++ b/ecs/docs/get-started-linux.md @@ -0,0 +1,84 @@ +Getting Started with Docker AWS ECS Plugin Beta on Linux +-------------------------------------------------------- + +The beta release of [AWS ECS](https://aws.amazon.com/ecs/) support for the +Docker CLI is shipped as a CLI plugin. Later releases will be included as part +of the Docker CLI. + +This plugin is included as part of Docker Desktop on Windows and macOS but on +Linux it needs to be installed manually. + +## Prerequisites + +* [Docker 19.03 or later](https://docs.docker.com/get-docker/) + +## Step by step install + +### Download + +You can download the Docker ECS plugin from this repository using the following +command: + + +```console +$ curl -L http://xxx | tar xzf - +``` + +You will then need to make it executable: + +```console +$ chmod +x docker-ecs +``` + +### Plugin install + +In order for the Docker CLI to use the downloaded plugin, you will need to move +it to the right place: + +```console +$ mkdir -p /usr/local/lib/docker/cli-plugins + +$ mv docker-ecs /usr/local/lib/docker/cli-plugins/ +``` + +You can put the CLI plugin into any of the following directories: + +* `/usr/local/lib/docker/cli-plugins` +* `/usr/local/libexec/docker/cli-plugins` +* `/usr/lib/docker/cli-plugins` +* `/usr/libexec/docker/cli-plugins` + +Finally you need to enable the experimental features on the CLI. This can be +done by setting the environment variable `DOCKER_CLI_EXPERIMENTAL=enabled` or by +setting `experimental` to `"enabled"` in your Docker config found at +`~/.docker/config.json`: + +```console +$ export DOCKER_CLI_EXPERIMENTAL=enabled + +$ DOCKER_CLI_EXPERIMENTAL=enabled docker help + +$ cat ~/.docker/config.json +{ + "experimental" : "enabled", + "auths" : { + "https://index.docker.io/v1/" : { + + } + } +} +``` + +To verify the CLI plugin installation, you can check that it appears in the CLI +help output or by outputting the plugin version: + +```console +$ docker help | grep ecs + ecs* Docker ECS (Docker Inc., 0.0.1) + +$ docker ecs version +Docker ECS plugin 0.0.1 +``` + + +You are now ready to [start deploying to ECS](http://xxx) From 4be356245048c6b7d13fd4eaed5a79dacffe2e71 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 10 Jun 2020 10:51:52 +0200 Subject: [PATCH 118/205] Revert "Refactor build process to build in containers" This reverts commit adab0d1bdf7bf2cc242128aae7f5044bd5182ea1. Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 32 -------------------------------- ecs/Makefile | 37 +++++++------------------------------ ecs/builder.Makefile | 39 --------------------------------------- 3 files changed, 7 insertions(+), 101 deletions(-) delete mode 100644 ecs/Dockerfile delete mode 100644 ecs/builder.Makefile diff --git a/ecs/Dockerfile b/ecs/Dockerfile deleted file mode 100644 index db374fb8..00000000 --- a/ecs/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# syntax = docker/dockerfile:experimental -ARG GO_VERSION=1.14.2 - -FROM golang:${GO_VERSION} AS base -ARG TARGET_OS=unknown -ARG TARGET_ARCH=unknown -ARG PWD=/ecs-plugin -ENV GO111MODULE=on - -WORKDIR ${PWD} -ADD go.* ${PWD} -RUN go mod download -ADD . ${PWD} - -FROM base AS make-plugin -RUN --mount=type=cache,target=/root/.cache/go-build \ - GOOS=${TARGET_OS} \ - GOARCH=${TARGET_ARCH} \ - make -f builder.Makefile build - -FROM base AS make-cross -RUN --mount=type=cache,target=/root/.cache/go-build \ - make -f builder.Makefile cross - -FROM scratch AS build -COPY --from=make-plugin /ecs-plugin/dist/* . - -FROM scratch AS cross -COPY --from=make-cross /ecs-plugin/dist/* . - -FROM base as test -RUN make -f builder.Makefile test diff --git a/ecs/Makefile b/ecs/Makefile index c5fe2875..0fd6b65a 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -1,27 +1,11 @@ -GOOS ?= $(shell go env GOOS) -GOARCH ?= $(shell go env GOARCH) -PWD = $(shell pwd) +clean: + rm -rf dist/ -export DOCKER_BUILDKIT=1 - -.DEFAULT_GOAL := build - -build: ## Build for the current - @docker build . \ - --output type=local,dest=./dist \ - --build-arg TARGET_OS=${GOOS} \ - --build-arg TARGET_ARCH=${GOARCH} \ - --target build - -cross: ## Cross build for linux, macos and windows - @docker build . \ - --output type=local,dest=./dist \ - --target cross +build: + go build -v -o dist/docker-ecs cmd/main/main.go test: build ## Run tests - @docker build . \ - --output type=local,dest=./dist \ - --target test + go test ./... -v e2e: build ## Run tests go test ./... -v -tags=e2e @@ -31,13 +15,6 @@ dev: build ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" lint: ## Verify Go files - @docker run --rm -t \ - -v $(PWD):/app \ - -w /app \ - golangci/golangci-lint:v1.27-alpine \ - golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... + golangci-lint run --config ./golangci.yaml ./... -clean: - rm -rf dist/ - -.PHONY: clean build test dev lint e2e cross +.PHONY: clean build test dev lint e2e diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile deleted file mode 100644 index 81145e02..00000000 --- a/ecs/builder.Makefile +++ /dev/null @@ -1,39 +0,0 @@ -GOOS ?= $(shell go env GOOS) -GOARCH ?= $(shell go env GOARCH) - -PROTOS=$(shell find . -name \*.proto) - -EXTENSION := -ifeq ($(GOOS),windows) - EXTENSION := .exe -endif - -STATIC_FLAGS= CGO_ENABLED=0 -LDFLAGS := "-s -w" -GO_BUILD = $(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) - -BINARY=dist/docker-ecs -BINARY_WITH_EXTENSION=$(BINARY)$(EXTENSION) - -export DOCKER_BUILDKIT=1 - -all: build - -clean: - rm -rf dist/ - -build: - $(GO_BUILD) -v -o $(BINARY_WITH_EXTENSION) cmd/main/main.go - -cross: - @GOOS=linux GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-linux-amd64 cmd/main/main.go - @GOOS=darwin GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-darwin-amd64 cmd/main/main.go - @GOOS=windows GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-windows-amd64.exe cmd/main/main.go - -test: build ## Run tests - @go test ./... -v - -lint: ## Verify Go files - golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... - -.PHONY: clean build test dev lint e2e From 1a09dc51ea16b85713b70c6aa04c792de6b8cf29 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 10 Jun 2020 15:07:25 +0200 Subject: [PATCH 119/205] Refactor build process to build in containers This is a re-apply from a previous commit Signed-off-by: Ulysses Souza Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 32 ++++++++++++++++++++++++++++++++ ecs/Makefile | 37 ++++++++++++++++++++++++++++++------- ecs/builder.Makefile | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 ecs/Dockerfile create mode 100644 ecs/builder.Makefile diff --git a/ecs/Dockerfile b/ecs/Dockerfile new file mode 100644 index 00000000..db374fb8 --- /dev/null +++ b/ecs/Dockerfile @@ -0,0 +1,32 @@ +# syntax = docker/dockerfile:experimental +ARG GO_VERSION=1.14.2 + +FROM golang:${GO_VERSION} AS base +ARG TARGET_OS=unknown +ARG TARGET_ARCH=unknown +ARG PWD=/ecs-plugin +ENV GO111MODULE=on + +WORKDIR ${PWD} +ADD go.* ${PWD} +RUN go mod download +ADD . ${PWD} + +FROM base AS make-plugin +RUN --mount=type=cache,target=/root/.cache/go-build \ + GOOS=${TARGET_OS} \ + GOARCH=${TARGET_ARCH} \ + make -f builder.Makefile build + +FROM base AS make-cross +RUN --mount=type=cache,target=/root/.cache/go-build \ + make -f builder.Makefile cross + +FROM scratch AS build +COPY --from=make-plugin /ecs-plugin/dist/* . + +FROM scratch AS cross +COPY --from=make-cross /ecs-plugin/dist/* . + +FROM base as test +RUN make -f builder.Makefile test diff --git a/ecs/Makefile b/ecs/Makefile index 0fd6b65a..c5fe2875 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -1,11 +1,27 @@ -clean: - rm -rf dist/ +GOOS ?= $(shell go env GOOS) +GOARCH ?= $(shell go env GOARCH) +PWD = $(shell pwd) -build: - go build -v -o dist/docker-ecs cmd/main/main.go +export DOCKER_BUILDKIT=1 + +.DEFAULT_GOAL := build + +build: ## Build for the current + @docker build . \ + --output type=local,dest=./dist \ + --build-arg TARGET_OS=${GOOS} \ + --build-arg TARGET_ARCH=${GOARCH} \ + --target build + +cross: ## Cross build for linux, macos and windows + @docker build . \ + --output type=local,dest=./dist \ + --target cross test: build ## Run tests - go test ./... -v + @docker build . \ + --output type=local,dest=./dist \ + --target test e2e: build ## Run tests go test ./... -v -tags=e2e @@ -15,6 +31,13 @@ dev: build ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" lint: ## Verify Go files - golangci-lint run --config ./golangci.yaml ./... + @docker run --rm -t \ + -v $(PWD):/app \ + -w /app \ + golangci/golangci-lint:v1.27-alpine \ + golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... -.PHONY: clean build test dev lint e2e +clean: + rm -rf dist/ + +.PHONY: clean build test dev lint e2e cross diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile new file mode 100644 index 00000000..0fd8e30b --- /dev/null +++ b/ecs/builder.Makefile @@ -0,0 +1,39 @@ +GOOS ?= $(shell go env GOOS) +GOARCH ?= $(shell go env GOARCH) + +PROTOS=$(shell find . -name \*.proto) + +EXTENSION := +ifeq ($(GOOS),windows) + EXTENSION := .exe +endif + +STATIC_FLAGS= CGO_ENABLED=0 +LDFLAGS := "-s -w" +GO_BUILD = $(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) + +BINARY=dist/docker +BINARY_WITH_EXTENSION=$(BINARY)$(EXTENSION) + +export DOCKER_BUILDKIT=1 + +all: build + +clean: + rm -rf dist/ + +build: + $(GO_BUILD) -v -o $(BINARY_WITH_EXTENSION) cmd/main/main.go + +cross: + @GOOS=linux GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-linux-amd64 cmd/main/main.go + @GOOS=darwin GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-darwin-amd64 cmd/main/main.go + @GOOS=windows GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-windows-amd64.exe cmd/main/main.go + +test: build ## Run tests + @go test ./... -v + +lint: ## Verify Go files + golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... + +.PHONY: clean build test dev lint e2e From a0500799d04ab52c311d894343312a40d5451eee Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 10 Jun 2020 15:16:11 +0200 Subject: [PATCH 120/205] Fix and optimize build process Kudos @chris-crone! Signed-off-by: Ulysses Souza Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 49 ++++++++++++++++++++++++++++------------ ecs/Makefile | 22 ++++++------------ ecs/builder.Makefile | 22 ++++++++---------- ecs/go.mod | 3 ++- ecs/go.sum | 16 +++++++++---- ecs/tests/plugin_test.go | 33 --------------------------- 6 files changed, 64 insertions(+), 81 deletions(-) delete mode 100644 ecs/tests/plugin_test.go diff --git a/ecs/Dockerfile b/ecs/Dockerfile index db374fb8..8d18ddfe 100644 --- a/ecs/Dockerfile +++ b/ecs/Dockerfile @@ -1,32 +1,51 @@ # syntax = docker/dockerfile:experimental -ARG GO_VERSION=1.14.2 +ARG GO_VERSION=1.14.4-alpine +ARG ALPINE_PKG_DOCKER_VERSION=19.03.11-r0 +ARG GOLANGCI_LINT_VERSION=v1.27.0-alpine -FROM golang:${GO_VERSION} AS base -ARG TARGET_OS=unknown -ARG TARGET_ARCH=unknown -ARG PWD=/ecs-plugin +FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS base +WORKDIR /ecs-plugin ENV GO111MODULE=on - -WORKDIR ${PWD} -ADD go.* ${PWD} -RUN go mod download -ADD . ${PWD} +ARG ALPINE_PKG_DOCKER_VERSION +RUN apk add --no-cache \ + docker=${ALPINE_PKG_DOCKER_VERSION} \ + make +COPY go.* . +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download +COPY . . FROM base AS make-plugin +ARG TARGETOS +ARG TARGETARCH RUN --mount=type=cache,target=/root/.cache/go-build \ - GOOS=${TARGET_OS} \ - GOARCH=${TARGET_ARCH} \ + --mount=type=cache,target=/go/pkg/mod \ + GOOS=${TARGETOS} \ + GOARCH=${TARGETARCH} \ make -f builder.Makefile build FROM base AS make-cross RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ make -f builder.Makefile cross FROM scratch AS build -COPY --from=make-plugin /ecs-plugin/dist/* . +COPY --from=make-plugin /ecs-plugin/dist/docker-ecs . FROM scratch AS cross COPY --from=make-cross /ecs-plugin/dist/* . -FROM base as test -RUN make -f builder.Makefile test +FROM make-plugin AS test +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + make -f builder.Makefile test + +FROM golangci/golangci-lint:${GOLANGCI_LINT_VERSION} AS lint-base + +FROM base AS lint +COPY --from=lint-base /usr/bin/golangci-lint /usr/bin/golangci-lint +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/golangci-lint \ + make -f builder.Makefile lint diff --git a/ecs/Makefile b/ecs/Makefile index c5fe2875..22dd3604 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -1,6 +1,5 @@ -GOOS ?= $(shell go env GOOS) -GOARCH ?= $(shell go env GOARCH) -PWD = $(shell pwd) +PLATFORM?=local +PWD=$(shell pwd) export DOCKER_BUILDKIT=1 @@ -8,20 +7,17 @@ export DOCKER_BUILDKIT=1 build: ## Build for the current @docker build . \ - --output type=local,dest=./dist \ - --build-arg TARGET_OS=${GOOS} \ - --build-arg TARGET_ARCH=${GOARCH} \ + --output ./dist \ + --platform ${PLATFORM} \ --target build cross: ## Cross build for linux, macos and windows @docker build . \ - --output type=local,dest=./dist \ + --output ./dist \ --target cross test: build ## Run tests - @docker build . \ - --output type=local,dest=./dist \ - --target test + @docker build . --target test e2e: build ## Run tests go test ./... -v -tags=e2e @@ -31,11 +27,7 @@ dev: build ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" lint: ## Verify Go files - @docker run --rm -t \ - -v $(PWD):/app \ - -w /app \ - golangci/golangci-lint:v1.27-alpine \ - golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... + @docker build . --target lint clean: rm -rf dist/ diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile index 0fd8e30b..8b6920ba 100644 --- a/ecs/builder.Makefile +++ b/ecs/builder.Makefile @@ -1,18 +1,16 @@ -GOOS ?= $(shell go env GOOS) -GOARCH ?= $(shell go env GOARCH) - -PROTOS=$(shell find . -name \*.proto) +GOOS?=$(shell go env GOOS) +GOARCH?=$(shell go env GOARCH) EXTENSION := ifeq ($(GOOS),windows) EXTENSION := .exe endif -STATIC_FLAGS= CGO_ENABLED=0 -LDFLAGS := "-s -w" -GO_BUILD = $(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) +STATIC_FLAGS=CGO_ENABLED=0 +LDFLAGS:="-s -w" +GO_BUILD=$(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) -BINARY=dist/docker +BINARY=dist/docker-ecs BINARY_WITH_EXTENSION=$(BINARY)$(EXTENSION) export DOCKER_BUILDKIT=1 @@ -30,10 +28,10 @@ cross: @GOOS=darwin GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-darwin-amd64 cmd/main/main.go @GOOS=windows GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-windows-amd64.exe cmd/main/main.go -test: build ## Run tests - @go test ./... -v +test: ## Run tests + @$(STATIC_FLAGS) go test -cover $(shell go list ./... | grep -vE 'e2e') lint: ## Verify Go files - golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... + $(STATIC_FLAGS) golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... -.PHONY: clean build test dev lint e2e +.PHONY: all clean build cross test dev lint diff --git a/ecs/go.mod b/ecs/go.mod index 7eedd06b..7dd8d7fe 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -40,6 +40,7 @@ require ( github.com/onsi/ginkgo v1.11.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/sirupsen/logrus v1.5.0 + github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/theupdateframework/notary v0.6.1 // indirect @@ -56,4 +57,4 @@ require ( vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect ) -go 1.13 +go 1.14 diff --git a/ecs/go.sum b/ecs/go.sum index 9088eaf2..f0303a30 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -19,9 +19,6 @@ github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkK github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0= -github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.30.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.30.22 h1:wImJ8jQrplgmxaTeUY7FrJFn4te/VtWq+mmmJ1TnWAg= github.com/aws/aws-sdk-go v1.30.22/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/awslabs/goformation/v4 v4.8.0 h1:UiUhyokRy3suEqBXTnipvY8klqY3Eyl4GCH17brraEc= @@ -45,9 +42,11 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= @@ -139,6 +138,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= @@ -162,14 +163,14 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -283,6 +284,10 @@ github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= @@ -405,6 +410,7 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= diff --git a/ecs/tests/plugin_test.go b/ecs/tests/plugin_test.go deleted file mode 100644 index 48d7e2f5..00000000 --- a/ecs/tests/plugin_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package tests - -import ( - "regexp" - "testing" - - "gotest.tools/assert" - "gotest.tools/v3/golden" - "gotest.tools/v3/icmd" -) - -func TestInvokePluginFromCLI(t *testing.T) { - cmd, cleanup, _ := dockerCli.createTestCmd() - defer cleanup() - // docker --help should list app as a top command - cmd.Command = dockerCli.Command("--help") - icmd.RunCmd(cmd).Assert(t, icmd.Expected{ - Out: "ecs* Docker ECS (Docker Inc.,", - }) - - // docker app --help prints docker-app help - cmd.Command = dockerCli.Command("ecs", "--help") - usage := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() - - goldenFile := "plugin-usage.golden" - golden.Assert(t, usage, goldenFile) - - // docker info should print app version and short description - cmd.Command = dockerCli.Command("info") - re := regexp.MustCompile(`ecs: Docker ECS \(Docker Inc\., .*\)`) - output := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() - assert.Assert(t, re.MatchString(output)) -} From d95798747153d76ff0cc1d4436948d515a43ebca Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 10 Jun 2020 10:42:48 +0200 Subject: [PATCH 121/205] Unit tests for cobra commands Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/root.go | 32 +++++++++++++++++ ecs/cmd/commands/root_test.go | 13 +++++++ ecs/cmd/commands/setup.go | 6 ---- ecs/cmd/commands/setup_test.go | 32 +++++++++++++++++ ecs/cmd/commands/testdata/context.golden | 1 + ecs/cmd/commands/version.go | 20 +++++++++++ ecs/cmd/commands/version_test.go | 18 ++++++++++ ecs/cmd/main/main.go | 44 ++--------------------- ecs/pkg/amazon/check_test.go | 2 +- ecs/pkg/amazon/cloudformation_test.go | 2 +- ecs/tests/command_test.go | 18 ---------- ecs/tests/e2e_deploy_services_test.go | 2 +- ecs/tests/setup_command_test.go | 24 ------------- ecs/tests/testdata/context-inspect.golden | 16 --------- 14 files changed, 122 insertions(+), 108 deletions(-) create mode 100644 ecs/cmd/commands/root.go create mode 100644 ecs/cmd/commands/root_test.go create mode 100644 ecs/cmd/commands/setup_test.go create mode 100644 ecs/cmd/commands/testdata/context.golden create mode 100644 ecs/cmd/commands/version.go create mode 100644 ecs/cmd/commands/version_test.go delete mode 100644 ecs/tests/command_test.go delete mode 100644 ecs/tests/setup_command_test.go delete mode 100644 ecs/tests/testdata/context-inspect.golden diff --git a/ecs/cmd/commands/root.go b/ecs/cmd/commands/root.go new file mode 100644 index 00000000..a9c22921 --- /dev/null +++ b/ecs/cmd/commands/root.go @@ -0,0 +1,32 @@ +package commands + +import ( + "fmt" + + "github.com/docker/cli/cli/command" + "github.com/spf13/cobra" +) + +// NewRootCmd returns the base root command. +func NewRootCmd(dockerCli command.Cli) *cobra.Command { + cmd := &cobra.Command{ + Short: "Docker ECS", + Long: `run multi-container applications on Amazon ECS.`, + Use: "ecs", + Annotations: map[string]string{"experimentalCLI": "true"}, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return fmt.Errorf("%q is not a docker ecs command\nSee 'docker ecs --help'", args[0]) + } + cmd.Help() + return nil + }, + } + cmd.AddCommand( + VersionCommand(), + ComposeCommand(dockerCli), + SecretCommand(dockerCli), + SetupCommand(), + ) + return cmd +} diff --git a/ecs/cmd/commands/root_test.go b/ecs/cmd/commands/root_test.go new file mode 100644 index 00000000..80f2a72d --- /dev/null +++ b/ecs/cmd/commands/root_test.go @@ -0,0 +1,13 @@ +package commands + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +func TestUnknownCommand(t *testing.T) { + root := NewRootCmd(nil) + _, _, err := root.Find([]string{"unknown_command"}) + assert.Error(t, err, "unknown command \"unknown_command\" for \"ecs\"") +} diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index dd6c40f6..6d52664f 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -9,7 +9,6 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/defaults" - "github.com/docker/cli/cli-plugins/plugin" contextStore "github.com/docker/ecs-plugin/pkg/docker" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -42,11 +41,6 @@ func SetupCommand() *cobra.Command { cmd := &cobra.Command{ Use: "setup", Short: "", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - //Override the root command PersistentPreRun - //We just need to initialize the top parent command - return plugin.PersistentPreRunE(cmd, args) - }, RunE: func(cmd *cobra.Command, args []string) error { if requiredFlag := opts.unsetRequiredArgs(); len(requiredFlag) > 0 { if err := interactiveCli(&opts); err != nil { diff --git a/ecs/cmd/commands/setup_test.go b/ecs/cmd/commands/setup_test.go new file mode 100644 index 00000000..97b7add3 --- /dev/null +++ b/ecs/cmd/commands/setup_test.go @@ -0,0 +1,32 @@ +package commands + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/docker/cli/cli/config" + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" + "gotest.tools/v3/golden" +) + +func TestDefaultAwsContextName(t *testing.T) { + dir := fs.NewDir(t, "setup") + defer dir.Remove() + cmd := NewRootCmd(nil) + dockerConfig := config.Dir() + config.SetDir(dir.Path()) + defer config.SetDir(dockerConfig) + + cmd.SetArgs([]string{"setup", "--cluster", "clusterName", "--profile", "profileName", "--region", "regionName"}) + err := cmd.Execute() + assert.NilError(t, err) + + files, err := filepath.Glob(dir.Join("contexts", "meta", "*", "meta.json")) + assert.NilError(t, err) + assert.Equal(t, len(files), 1) + b, err := ioutil.ReadFile(files[0]) + assert.NilError(t, err) + golden.Assert(t, string(b), "context.golden") +} diff --git a/ecs/cmd/commands/testdata/context.golden b/ecs/cmd/commands/testdata/context.golden new file mode 100644 index 00000000..891cb9cf --- /dev/null +++ b/ecs/cmd/commands/testdata/context.golden @@ -0,0 +1 @@ +{"Name":"aws","Metadata":{"Type":"aws"},"Endpoints":{"aws":{"Profile":"profileName","Cluster":"clusterName","Region":"regionName"},"docker":{"Profile":"profileName","Cluster":"clusterName","Region":"regionName"}}} \ No newline at end of file diff --git a/ecs/cmd/commands/version.go b/ecs/cmd/commands/version.go new file mode 100644 index 00000000..d3b44687 --- /dev/null +++ b/ecs/cmd/commands/version.go @@ -0,0 +1,20 @@ +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +const Version = "0.0.1" + +func VersionCommand() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Show version.", + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Fprintf(cmd.OutOrStdout(), "Docker ECS plugin %s\n", Version) + return nil + }, + } +} diff --git a/ecs/cmd/commands/version_test.go b/ecs/cmd/commands/version_test.go new file mode 100644 index 00000000..4c0ed7e6 --- /dev/null +++ b/ecs/cmd/commands/version_test.go @@ -0,0 +1,18 @@ +package commands + +import ( + "bytes" + "strings" + "testing" + + "gotest.tools/v3/assert" +) + +func TestVersion(t *testing.T) { + root := NewRootCmd(nil) + var out bytes.Buffer + root.SetOut(&out) + root.SetArgs([]string{"version"}) + root.Execute() + assert.Check(t, strings.Contains(out.String(), Version)) +} diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 9af64829..5dd1f4dd 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -1,60 +1,22 @@ package main import ( - "fmt" + "github.com/docker/ecs-plugin/cmd/commands" "github.com/docker/cli/cli-plugins/manager" "github.com/docker/cli/cli-plugins/plugin" "github.com/docker/cli/cli/command" - commands "github.com/docker/ecs-plugin/cmd/commands" "github.com/spf13/cobra" ) -const version = "0.0.1" - func main() { plugin.Run(func(dockerCli command.Cli) *cobra.Command { - cmd := NewRootCmd("ecs", dockerCli) + cmd := commands.NewRootCmd(dockerCli) return cmd }, manager.Metadata{ SchemaVersion: "0.1.0", Vendor: "Docker Inc.", - Version: version, + Version: commands.Version, Experimental: true, }) } - -// NewRootCmd returns the base root command. -func NewRootCmd(name string, dockerCli command.Cli) *cobra.Command { - cmd := &cobra.Command{ - Short: "Docker ECS", - Long: `run multi-container applications on Amazon ECS.`, - Use: name, - Annotations: map[string]string{"experimentalCLI": "true"}, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 0 { - return fmt.Errorf("%q is not a docker ecs command\nSee 'docker ecs --help'", args[0]) - } - cmd.Help() - return nil - }, - } - cmd.AddCommand( - VersionCommand(), - commands.ComposeCommand(dockerCli), - commands.SecretCommand(dockerCli), - commands.SetupCommand(), - ) - return cmd -} - -func VersionCommand() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Show version.", - RunE: func(cmd *cobra.Command, args []string) error { - fmt.Printf("Docker ECS plugin %s\n", version) - return nil - }, - } -} diff --git a/ecs/pkg/amazon/check_test.go b/ecs/pkg/amazon/check_test.go index 3f185954..3038eee8 100644 --- a/ecs/pkg/amazon/check_test.go +++ b/ecs/pkg/amazon/check_test.go @@ -3,7 +3,7 @@ package amazon import ( "testing" - "gotest.tools/assert" + "gotest.tools/v3/assert" ) func TestInvalidNetworkMode(t *testing.T) { diff --git a/ecs/pkg/amazon/cloudformation_test.go b/ecs/pkg/amazon/cloudformation_test.go index 84e12e76..4b525d8f 100644 --- a/ecs/pkg/amazon/cloudformation_test.go +++ b/ecs/pkg/amazon/cloudformation_test.go @@ -13,7 +13,7 @@ import ( "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" - "gotest.tools/assert" + "gotest.tools/v3/assert" "gotest.tools/v3/golden" ) diff --git a/ecs/tests/command_test.go b/ecs/tests/command_test.go deleted file mode 100644 index 3ca7433d..00000000 --- a/ecs/tests/command_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package tests - -import ( - "testing" - - "gotest.tools/v3/icmd" -) - -func TestExitErrorCode(t *testing.T) { - cmd, cleanup, _ := dockerCli.createTestCmd() - defer cleanup() - - cmd.Command = dockerCli.Command("ecs", "unknown_command") - icmd.RunCmd(cmd).Assert(t, icmd.Expected{ - ExitCode: 1, - Err: "\"unknown_command\" is not a docker ecs command\nSee 'docker ecs --help'", - }) -} diff --git a/ecs/tests/e2e_deploy_services_test.go b/ecs/tests/e2e_deploy_services_test.go index c7002a19..c8a2242c 100644 --- a/ecs/tests/e2e_deploy_services_test.go +++ b/ecs/tests/e2e_deploy_services_test.go @@ -10,7 +10,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/docker/ecs-plugin/pkg/amazon" "github.com/docker/ecs-plugin/pkg/docker" - "gotest.tools/assert" + "gotest.tools/v3/assert" "gotest.tools/v3/fs" "gotest.tools/v3/golden" "gotest.tools/v3/icmd" diff --git a/ecs/tests/setup_command_test.go b/ecs/tests/setup_command_test.go deleted file mode 100644 index d36393f4..00000000 --- a/ecs/tests/setup_command_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package tests - -import ( - "strings" - "testing" - - "gotest.tools/assert" - "gotest.tools/v3/golden" - "gotest.tools/v3/icmd" -) - -func TestDefaultAwsContextName(t *testing.T) { - cmd, cleanup, _ := dockerCli.createTestCmd() - defer cleanup() - - cmd.Command = dockerCli.Command("ecs", "setup", "--cluster", "clusterName", "--profile", "profileName", - "--region", "regionName") - icmd.RunCmd(cmd).Assert(t, icmd.Success) - - cmd.Command = dockerCli.Command("context", "inspect", "aws") - output := icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined() - expected := golden.Get(t, "context-inspect.golden") - assert.Assert(t, strings.HasPrefix(output, string(expected))) -} diff --git a/ecs/tests/testdata/context-inspect.golden b/ecs/tests/testdata/context-inspect.golden deleted file mode 100644 index ff61b55f..00000000 --- a/ecs/tests/testdata/context-inspect.golden +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "Name": "aws", - "Metadata": {}, - "Endpoints": { - "aws": { - "Cluster": "clusterName", - "Profile": "profileName", - "Region": "regionName" - }, - "docker": { - "SkipTLSVerify": false - } - }, - "TLSMaterial": {}, - "Storage": \ No newline at end of file From 0f1a362664edfc927f6d3b811fecff4175a47ca7 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 15 Jun 2020 09:27:12 +0200 Subject: [PATCH 122/205] Set FailureThreshold Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/cloudformation.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/cloudformation.go index 6afa06c3..51eba2c4 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/cloudformation.go @@ -298,8 +298,11 @@ func (c client) createServiceRegistry(service types.ServiceConfig, template *clo template.Resources[serviceRegistration] = &cloudmap.Service{ Description: fmt.Sprintf("%q service discovery entry in Cloud Map", service.Name), HealthCheckConfig: healthCheck, - Name: service.Name, - NamespaceId: cloudformation.Ref("CloudMap"), + HealthCheckCustomConfig: &cloudmap.Service_HealthCheckCustomConfig{ + FailureThreshold: 1, + }, + Name: service.Name, + NamespaceId: cloudformation.Ref("CloudMap"), DnsConfig: &cloudmap.Service_DnsConfig{ DnsRecords: []cloudmap.Service_DnsRecord{ { From d36b9b104e0576e79ebb787d4cadf9e6a6e7d74f Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 15 Jun 2020 09:41:01 +0200 Subject: [PATCH 123/205] Fix broken master Signed-off-by: Nicolas De Loof --- .../testdata/simple/simple-cloudformation-conversion.golden | 3 +++ .../simple-cloudformation-with-overrides-conversion.golden | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden index 7c7af678..0003b38e 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden @@ -143,6 +143,9 @@ ], "RoutingPolicy": "MULTIVALUE" }, + "HealthCheckCustomConfig": { + "FailureThreshold": 1 + }, "Name": "simple", "NamespaceId": { "Ref": "CloudMap" diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index e9535e90..07c47765 100644 --- a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -143,6 +143,9 @@ ], "RoutingPolicy": "MULTIVALUE" }, + "HealthCheckCustomConfig": { + "FailureThreshold": 1 + }, "Name": "simple", "NamespaceId": { "Ref": "CloudMap" From bb98dae0829a3cc0804d24cb5c5785503a5bbd6c Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 11 Jun 2020 19:22:12 +0200 Subject: [PATCH 124/205] code restructure Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 36 ++++-- ecs/cmd/commands/secret.go | 23 ++-- ecs/pkg/amazon/amazon.go | 8 ++ ecs/pkg/amazon/api.go | 11 -- .../amazon/{client.go => backend/backend.go} | 16 ++- .../amazon/{ => backend}/cloudformation.go | 56 +++++---- .../{ => backend}/cloudformation_test.go | 7 +- ecs/pkg/amazon/backend/down.go | 31 +++++ ecs/pkg/amazon/{ => backend}/down_test.go | 16 +-- ecs/pkg/amazon/{ => backend}/iam.go | 2 +- ecs/pkg/amazon/backend/list.go | 63 ++++++++++ ecs/pkg/amazon/{ => backend}/logs.go | 20 +-- ecs/pkg/amazon/backend/secrets.go | 23 ++++ .../simple-single-service-with-overrides.yaml | 0 .../testdata/input/simple-single-service.yaml | 0 .../testdata/invalid_network_mode.yaml | 0 .../simple-cloudformation-conversion.golden | 0 ...formation-with-overrides-conversion.golden | 0 ecs/pkg/amazon/backend/up.go | 100 +++++++++++++++ ecs/pkg/amazon/{ => backend}/wait.go | 22 +--- ecs/pkg/amazon/check_test.go | 13 -- ecs/pkg/amazon/{ => compatibility}/check.go | 2 +- ecs/pkg/amazon/compatibility/check_test.go | 23 ++++ .../{ => compatibility}/compatibility.go | 2 +- ecs/pkg/amazon/down.go | 34 ------ ecs/pkg/amazon/list.go | 80 ------------ ecs/pkg/amazon/sdk/api.go | 61 ++++++++++ ecs/pkg/amazon/{ => sdk}/api_mock.go | 20 +-- ecs/pkg/amazon/{ => sdk}/convert.go | 5 +- ecs/pkg/amazon/{ => sdk}/sdk.go | 33 ++--- ecs/pkg/amazon/secrets.go | 30 ----- .../secret.go => amazon/types/types.go} | 22 +++- ecs/pkg/amazon/{ => types}/x.go | 2 +- ecs/pkg/amazon/up.go | 114 ------------------ ecs/pkg/compose/api.go | 10 +- 35 files changed, 464 insertions(+), 421 deletions(-) create mode 100644 ecs/pkg/amazon/amazon.go delete mode 100644 ecs/pkg/amazon/api.go rename ecs/pkg/amazon/{client.go => backend/backend.go} (66%) rename ecs/pkg/amazon/{ => backend}/cloudformation.go (85%) rename ecs/pkg/amazon/{ => backend}/cloudformation_test.go (97%) create mode 100644 ecs/pkg/amazon/backend/down.go rename ecs/pkg/amazon/{ => backend}/down_test.go (73%) rename ecs/pkg/amazon/{ => backend}/iam.go (98%) create mode 100644 ecs/pkg/amazon/backend/list.go rename ecs/pkg/amazon/{ => backend}/logs.go (73%) create mode 100644 ecs/pkg/amazon/backend/secrets.go rename ecs/pkg/amazon/{ => backend}/testdata/input/simple-single-service-with-overrides.yaml (100%) rename ecs/pkg/amazon/{ => backend}/testdata/input/simple-single-service.yaml (100%) rename ecs/pkg/amazon/{ => backend}/testdata/invalid_network_mode.yaml (100%) rename ecs/pkg/amazon/{ => backend}/testdata/simple/simple-cloudformation-conversion.golden (100%) rename ecs/pkg/amazon/{ => backend}/testdata/simple/simple-cloudformation-with-overrides-conversion.golden (100%) create mode 100644 ecs/pkg/amazon/backend/up.go rename ecs/pkg/amazon/{ => backend}/wait.go (67%) delete mode 100644 ecs/pkg/amazon/check_test.go rename ecs/pkg/amazon/{ => compatibility}/check.go (98%) create mode 100644 ecs/pkg/amazon/compatibility/check_test.go rename ecs/pkg/amazon/{ => compatibility}/compatibility.go (99%) delete mode 100644 ecs/pkg/amazon/down.go delete mode 100644 ecs/pkg/amazon/list.go create mode 100644 ecs/pkg/amazon/sdk/api.go rename ecs/pkg/amazon/{ => sdk}/api_mock.go (96%) rename ecs/pkg/amazon/{ => sdk}/convert.go (98%) rename ecs/pkg/amazon/{ => sdk}/sdk.go (94%) delete mode 100644 ecs/pkg/amazon/secrets.go rename ecs/pkg/{docker/secret.go => amazon/types/types.go} (71%) rename ecs/pkg/amazon/{ => types}/x.go (93%) delete mode 100644 ecs/pkg/amazon/up.go diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 0b1e70e6..55eb7a14 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -3,9 +3,12 @@ package commands import ( "context" "fmt" + "io" + "os" + "strings" "github.com/docker/cli/cli/command" - "github.com/docker/ecs-plugin/pkg/amazon" + amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" "github.com/docker/ecs-plugin/pkg/compose" "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" @@ -47,11 +50,11 @@ func ConvertCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) if err != nil { return err } - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } - template, err := client.Convert(project) + template, err := backend.Convert(project) if err != nil { return err } @@ -77,11 +80,11 @@ func UpCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobr if err != nil { return err } - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } - return client.ComposeUp(context.Background(), project) + return backend.ComposeUp(context.Background(), project) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") @@ -97,11 +100,20 @@ func PsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobr if err != nil { return err } - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } - return client.ComposePs(context.Background(), project) + tasks, err := backend.ComposePs(context.Background(), project) + if err != nil { + return err + } + printSection(os.Stdout, len(tasks), func(w io.Writer) { + for _, task := range tasks { + fmt.Fprintf(w, "%s\t%s\t%s\n", task.Name, task.State, strings.Join(task.Ports, " ")) + } + }, "NAME", "STATE", "PORTS") + return nil }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") @@ -117,7 +129,7 @@ func DownCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co cmd := &cobra.Command{ Use: "down", RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } @@ -126,11 +138,11 @@ func DownCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co if err != nil { return err } - return client.ComposeDown(context.Background(), project.Name, opts.DeleteCluster) + return backend.ComposeDown(context.Background(), project.Name, opts.DeleteCluster) } // project names passed as parameters for _, name := range args { - err := client.ComposeDown(context.Background(), name, opts.DeleteCluster) + err := backend.ComposeDown(context.Background(), name, opts.DeleteCluster) if err != nil { return err } @@ -146,7 +158,7 @@ func LogsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co cmd := &cobra.Command{ Use: "logs [PROJECT NAME]", RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } @@ -161,7 +173,7 @@ func LogsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co } else { name = args[0] } - return client.ComposeLogs(context.Background(), name) + return backend.ComposeLogs(context.Background(), name) }), } return cmd diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go index f964f1ea..a426740d 100644 --- a/ecs/cmd/commands/secret.go +++ b/ecs/cmd/commands/secret.go @@ -10,7 +10,8 @@ import ( "text/tabwriter" "github.com/docker/cli/cli/command" - "github.com/docker/ecs-plugin/pkg/amazon" + amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" + "github.com/docker/ecs-plugin/pkg/amazon/types" "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" ) @@ -47,7 +48,7 @@ func CreateSecret(dockerCli command.Cli) *cobra.Command { Use: "create NAME", Short: "Creates a secret.", RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } @@ -56,8 +57,8 @@ func CreateSecret(dockerCli command.Cli) *cobra.Command { } name := args[0] - secret := docker.NewSecret(name, opts.Username, opts.Password, opts.Description) - id, err := client.CreateSecret(context.Background(), secret) + secret := types.NewSecret(name, opts.Username, opts.Password, opts.Description) + id, err := backend.CreateSecret(context.Background(), secret) fmt.Println(id) return err }), @@ -73,7 +74,7 @@ func InspectSecret(dockerCli command.Cli) *cobra.Command { Use: "inspect ID", Short: "Displays secret details", RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } @@ -81,7 +82,7 @@ func InspectSecret(dockerCli command.Cli) *cobra.Command { return errors.New("Missing mandatory parameter: ID") } id := args[0] - secret, err := client.InspectSecret(context.Background(), id) + secret, err := backend.InspectSecret(context.Background(), id) if err != nil { return err } @@ -102,11 +103,11 @@ func ListSecrets(dockerCli command.Cli) *cobra.Command { Aliases: []string{"ls"}, Short: "List secrets stored for the existing account.", RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } - secrets, err := client.ListSecrets(context.Background()) + secrets, err := backend.ListSecrets(context.Background()) if err != nil { return err } @@ -125,21 +126,21 @@ func DeleteSecret(dockerCli command.Cli) *cobra.Command { Aliases: []string{"rm", "remove"}, Short: "Removes a secret.", RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - client, err := amazon.NewClient(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } if len(args) == 0 { return errors.New("Missing mandatory parameter: [NAME]") } - return client.DeleteSecret(context.Background(), args[0], opts.recover) + return backend.DeleteSecret(context.Background(), args[0], opts.recover) }), } cmd.Flags().BoolVar(&opts.recover, "recover", false, "Enable recovery.") return cmd } -func printList(out io.Writer, secrets []docker.Secret) { +func printList(out io.Writer, secrets []types.Secret) { printSection(out, len(secrets), func(w io.Writer) { for _, secret := range secrets { fmt.Fprintf(w, "%s\t%s\t%s\n", secret.ID, secret.Name, secret.Description) diff --git a/ecs/pkg/amazon/amazon.go b/ecs/pkg/amazon/amazon.go new file mode 100644 index 00000000..f6c588d1 --- /dev/null +++ b/ecs/pkg/amazon/amazon.go @@ -0,0 +1,8 @@ +package amazon + +import ( + "github.com/docker/ecs-plugin/pkg/amazon/backend" + "github.com/docker/ecs-plugin/pkg/compose" +) + +var _ compose.API = &backend.Backend{} diff --git a/ecs/pkg/amazon/api.go b/ecs/pkg/amazon/api.go deleted file mode 100644 index 5b058584..00000000 --- a/ecs/pkg/amazon/api.go +++ /dev/null @@ -1,11 +0,0 @@ -package amazon - -//go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon" -package=amazon . API - -type API interface { - downAPI - upAPI - logsAPI - secretsAPI - listAPI -} diff --git a/ecs/pkg/amazon/client.go b/ecs/pkg/amazon/backend/backend.go similarity index 66% rename from ecs/pkg/amazon/client.go rename to ecs/pkg/amazon/backend/backend.go index 839c5118..0d2f07f9 100644 --- a/ecs/pkg/amazon/client.go +++ b/ecs/pkg/amazon/backend/backend.go @@ -1,9 +1,9 @@ -package amazon +package backend import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/amazon/sdk" ) const ( @@ -12,7 +12,7 @@ const ( ServiceTag = "com.docker.compose.service" ) -func NewClient(profile string, cluster string, region string) (compose.API, error) { +func NewBackend(profile string, cluster string, region string) (*Backend, error) { sess, err := session.NewSessionWithOptions(session.Options{ Profile: profile, Config: aws.Config{ @@ -22,17 +22,15 @@ func NewClient(profile string, cluster string, region string) (compose.API, erro if err != nil { return nil, err } - return &client{ + return &Backend{ Cluster: cluster, Region: region, - api: NewAPI(sess), + api: sdk.NewAPI(sess), }, nil } -type client struct { +type Backend struct { Cluster string Region string - api API + api sdk.API } - -var _ compose.API = &client{} diff --git a/ecs/pkg/amazon/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go similarity index 85% rename from ecs/pkg/amazon/cloudformation.go rename to ecs/pkg/amazon/backend/cloudformation.go index 51eba2c4..115a67d6 100644 --- a/ecs/pkg/amazon/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -1,4 +1,4 @@ -package amazon +package backend import ( "fmt" @@ -21,6 +21,9 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/logs" cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" "github.com/awslabs/goformation/v4/cloudformation/tags" + "github.com/docker/ecs-plugin/pkg/amazon/compatibility" + sdk "github.com/docker/ecs-plugin/pkg/amazon/sdk" + btypes "github.com/docker/ecs-plugin/pkg/amazon/types" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -33,8 +36,8 @@ const ( ) // Convert a compose project into a CloudFormation template -func (c client) Convert(project *compose.Project) (*cloudformation.Template, error) { - warnings := Check(project) +func (b Backend) Convert(project *compose.Project) (*cloudformation.Template, error) { + warnings := compatibility.Check(project) for _, w := range warnings { logrus.Warn(w) } @@ -75,7 +78,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err // Create Cluster is `ParameterClusterName` parameter is not set template.Conditions["CreateCluster"] = cloudformation.Equals("", cloudformation.Ref(ParameterClusterName)) - cluster := c.createCluster(project, template) + cluster := createCluster(project, template) networks := map[string]string{} for _, net := range project.Networks { @@ -88,17 +91,18 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err } // Private DNS namespace will allow DNS name for the services to be ..local - c.createCloudMap(project, template) + createCloudMap(project, template) - loadBalancerARN := c.createLoadBalancer(project, template) + loadBalancerARN := createLoadBalancer(project, template) for _, service := range project.Services { - definition, err := Convert(project, service) + + definition, err := sdk.Convert(project, service) if err != nil { return nil, err } - taskExecutionRole, err := c.createTaskExecutionRole(service, err, definition, template) + taskExecutionRole, err := createTaskExecutionRole(service, err, definition, template) if err != nil { return template, err } @@ -112,7 +116,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err // FIXME ECS only support HTTP(s) health checks, while Docker only support CMD } - serviceRegistry := c.createServiceRegistry(service, template, healthCheck) + serviceRegistry := createServiceRegistry(service, template, healthCheck) serviceSecurityGroups := []string{} for net := range service.Networks { @@ -124,14 +128,14 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err if len(service.Ports) > 0 { for _, port := range service.Ports { protocol := strings.ToUpper(port.Protocol) - if c.getLoadBalancerType(project) == elbv2.LoadBalancerTypeEnumApplication { + if getLoadBalancerType(project) == elbv2.LoadBalancerTypeEnumApplication { protocol = elbv2.ProtocolEnumHttps if port.Published == 80 { protocol = elbv2.ProtocolEnumHttp } } - targetGroupName := c.createTargetGroup(project, service, port, template, protocol) - listenerName := c.createListener(service, port, template, targetGroupName, loadBalancerARN, protocol) + targetGroupName := createTargetGroup(project, service, port, template, protocol) + listenerName := createListener(service, port, template, targetGroupName, loadBalancerARN, protocol) dependsOn = append(dependsOn, listenerName) serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ ContainerName: service.Name, @@ -184,7 +188,7 @@ func (c client) Convert(project *compose.Project) (*cloudformation.Template, err return template, nil } -func (c client) getLoadBalancerType(project *compose.Project) string { +func getLoadBalancerType(project *compose.Project) string { for _, service := range project.Services { for _, port := range service.Ports { if port.Published != 80 && port.Published != 443 { @@ -195,7 +199,7 @@ func (c client) getLoadBalancerType(project *compose.Project) string { return elbv2.LoadBalancerTypeEnumApplication } -func (c client) getLoadBalancerSecurityGroups(project *compose.Project, template *cloudformation.Template) []string { +func getLoadBalancerSecurityGroups(project *compose.Project, template *cloudformation.Template) []string { securityGroups := []string{} for _, network := range project.Networks { if !network.Internal { @@ -206,15 +210,15 @@ func (c client) getLoadBalancerSecurityGroups(project *compose.Project, template return uniqueStrings(securityGroups) } -func (c client) createLoadBalancer(project *compose.Project, template *cloudformation.Template) string { +func createLoadBalancer(project *compose.Project, template *cloudformation.Template) string { loadBalancerName := fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name)) // Create LoadBalancer if `ParameterLoadBalancerName` is not set template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) - loadBalancerType := c.getLoadBalancerType(project) + loadBalancerType := getLoadBalancerType(project) securityGroups := []string{} if loadBalancerType == elbv2.LoadBalancerTypeEnumApplication { - securityGroups = c.getLoadBalancerSecurityGroups(project, template) + securityGroups = getLoadBalancerSecurityGroups(project, template) } template.Resources[loadBalancerName] = &elasticloadbalancingv2.LoadBalancer{ @@ -237,7 +241,7 @@ func (c client) createLoadBalancer(project *compose.Project, template *cloudform return cloudformation.If("CreateLoadBalancer", cloudformation.Ref(loadBalancerName), cloudformation.Ref(ParameterLoadBalancerARN)) } -func (c client) createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerARN string, protocol string) string { +func createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerARN string, protocol string) string { listenerName := fmt.Sprintf( "%s%s%dListener", normalizeResourceName(service.Name), @@ -266,7 +270,7 @@ func (c client) createListener(service types.ServiceConfig, port types.ServicePo return listenerName } -func (c client) createTargetGroup(project *compose.Project, service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, protocol string) string { +func createTargetGroup(project *compose.Project, service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, protocol string) string { targetGroupName := fmt.Sprintf( "%s%s%dTargetGroup", normalizeResourceName(service.Name), @@ -289,7 +293,7 @@ func (c client) createTargetGroup(project *compose.Project, service types.Servic return targetGroupName } -func (c client) createServiceRegistry(service types.ServiceConfig, template *cloudformation.Template, healthCheck *cloudmap.Service_HealthCheckConfig) ecs.Service_ServiceRegistry { +func createServiceRegistry(service types.ServiceConfig, template *cloudformation.Template, healthCheck *cloudmap.Service_HealthCheckConfig) ecs.Service_ServiceRegistry { serviceRegistration := fmt.Sprintf("%sServiceDiscoveryEntry", normalizeResourceName(service.Name)) serviceRegistry := ecs.Service_ServiceRegistry{ RegistryArn: cloudformation.GetAtt(serviceRegistration, "Arn"), @@ -316,9 +320,9 @@ func (c client) createServiceRegistry(service types.ServiceConfig, template *clo return serviceRegistry } -func (c client) createTaskExecutionRole(service types.ServiceConfig, err error, definition *ecs.TaskDefinition, template *cloudformation.Template) (string, error) { +func createTaskExecutionRole(service types.ServiceConfig, err error, definition *ecs.TaskDefinition, template *cloudformation.Template) (string, error) { taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", normalizeResourceName(service.Name)) - policy, err := c.getPolicy(definition) + policy, err := getPolicy(definition) if err != nil { return taskExecutionRole, err } @@ -341,7 +345,7 @@ func (c client) createTaskExecutionRole(service types.ServiceConfig, err error, return taskExecutionRole, nil } -func (c client) createCluster(project *compose.Project, template *cloudformation.Template) string { +func createCluster(project *compose.Project, template *cloudformation.Template) string { template.Resources["Cluster"] = &ecs.Cluster{ ClusterName: project.Name, Tags: []tags.Tag{ @@ -356,7 +360,7 @@ func (c client) createCluster(project *compose.Project, template *cloudformation return cluster } -func (c client) createCloudMap(project *compose.Project, template *cloudformation.Template) { +func createCloudMap(project *compose.Project, template *cloudformation.Template) { template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), Name: fmt.Sprintf("%s.local", project.Name), @@ -365,7 +369,7 @@ func (c client) createCloudMap(project *compose.Project, template *cloudformatio } func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc string, template *cloudformation.Template) string { - if sg, ok := net.Extras[ExtensionSecurityGroup]; ok { + if sg, ok := net.Extras[btypes.ExtensionSecurityGroup]; ok { logrus.Debugf("Security Group for network %q set by user to %q", net.Name, sg) return sg.(string) } @@ -428,7 +432,7 @@ func normalizeResourceName(s string) string { return strings.Title(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(s, "")) } -func (c client) getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { +func getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { arns := []string{} for _, container := range taskDef.ContainerDefinitions { if container.RepositoryCredentials != nil { diff --git a/ecs/pkg/amazon/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go similarity index 97% rename from ecs/pkg/amazon/cloudformation_test.go rename to ecs/pkg/amazon/backend/cloudformation_test.go index 4b525d8f..12271478 100644 --- a/ecs/pkg/amazon/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -1,4 +1,4 @@ -package amazon +package backend import ( "fmt" @@ -13,6 +13,7 @@ import ( "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" + "gotest.tools/v3/assert" "gotest.tools/v3/golden" ) @@ -103,7 +104,7 @@ services: } func convertResultAsString(t *testing.T, project *compose.Project, clusterName string) string { - client, err := NewClient("", clusterName, "") + client, err := NewBackend("", clusterName, "") assert.NilError(t, err) result, err := client.Convert(project) assert.NilError(t, err) @@ -133,7 +134,7 @@ func convertYaml(t *testing.T, yaml string) *cloudformation.Template { assert.NilError(t, err) err = compose.Normalize(model) assert.NilError(t, err) - template, err := client{}.Convert(&compose.Project{ + template, err := Backend{}.Convert(&compose.Project{ Config: *model, Name: "test", }) diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go new file mode 100644 index 00000000..8b978e79 --- /dev/null +++ b/ecs/pkg/amazon/backend/down.go @@ -0,0 +1,31 @@ +package backend + +import ( + "context" + "fmt" + + "github.com/docker/ecs-plugin/pkg/amazon/types" +) + +func (b *Backend) ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error { + err := b.api.DeleteStack(ctx, projectName) + if err != nil { + return err + } + + err = b.WaitStackCompletion(ctx, projectName, types.StackDelete) + if err != nil { + return err + } + + if !deleteCluster { + return nil + } + + fmt.Printf("Delete cluster %s", b.Cluster) + if err = b.api.DeleteCluster(ctx, b.Cluster); err != nil { + return err + } + fmt.Printf("... done. \n") + return nil +} diff --git a/ecs/pkg/amazon/down_test.go b/ecs/pkg/amazon/backend/down_test.go similarity index 73% rename from ecs/pkg/amazon/down_test.go rename to ecs/pkg/amazon/backend/down_test.go index 642faf75..d9abf70c 100644 --- a/ecs/pkg/amazon/down_test.go +++ b/ecs/pkg/amazon/backend/down_test.go @@ -1,17 +1,19 @@ -package amazon +package backend import ( "context" "testing" + "github.com/docker/ecs-plugin/pkg/amazon/sdk" + btypes "github.com/docker/ecs-plugin/pkg/amazon/types" "github.com/golang/mock/gomock" ) func TestDownDontDeleteCluster(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - m := NewMockAPI(ctrl) - c := &client{ + m := sdk.NewMockAPI(ctrl) + c := &Backend{ Cluster: "test_cluster", Region: "region", api: m, @@ -20,7 +22,7 @@ func TestDownDontDeleteCluster(t *testing.T) { recorder := m.EXPECT() recorder.DeleteStack(ctx, "test_project").Return(nil) recorder.GetStackID(ctx, "test_project").Return("stack-123", nil) - recorder.WaitStackComplete(ctx, "stack-123", StackDelete).Return(nil) + recorder.WaitStackComplete(ctx, "stack-123", btypes.StackDelete).Return(nil) recorder.DescribeStackEvents(ctx, "stack-123").Return(nil, nil) c.ComposeDown(ctx, "test_project", false) @@ -29,8 +31,8 @@ func TestDownDontDeleteCluster(t *testing.T) { func TestDownDeleteCluster(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - m := NewMockAPI(ctrl) - c := &client{ + m := sdk.NewMockAPI(ctrl) + c := &Backend{ Cluster: "test_cluster", Region: "region", api: m, @@ -40,7 +42,7 @@ func TestDownDeleteCluster(t *testing.T) { recorder := m.EXPECT() recorder.DeleteStack(ctx, "test_project").Return(nil) recorder.GetStackID(ctx, "test_project").Return("stack-123", nil) - recorder.WaitStackComplete(ctx, "stack-123", StackDelete).Return(nil) + recorder.WaitStackComplete(ctx, "stack-123", btypes.StackDelete).Return(nil) recorder.DescribeStackEvents(ctx, "stack-123").Return(nil, nil) recorder.DeleteCluster(ctx, "test_cluster").Return(nil) diff --git a/ecs/pkg/amazon/iam.go b/ecs/pkg/amazon/backend/iam.go similarity index 98% rename from ecs/pkg/amazon/iam.go rename to ecs/pkg/amazon/backend/iam.go index affcaaae..81a4fdb0 100644 --- a/ecs/pkg/amazon/iam.go +++ b/ecs/pkg/amazon/backend/iam.go @@ -1,4 +1,4 @@ -package amazon +package backend const ( ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go new file mode 100644 index 00000000..055362aa --- /dev/null +++ b/ecs/pkg/amazon/backend/list.go @@ -0,0 +1,63 @@ +package backend + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/docker/ecs-plugin/pkg/compose" +) + +func (b *Backend) ComposePs(ctx context.Context, project *compose.Project) ([]types.TaskStatus, error) { + cluster := b.Cluster + if cluster == "" { + cluster = project.Name + } + arns := []string{} + for _, service := range project.Services { + tasks, err := b.api.ListTasks(ctx, cluster, service.Name) + if err != nil { + return []types.TaskStatus{}, err + } + arns = append(arns, tasks...) + } + if len(arns) == 0 { + return []types.TaskStatus{}, nil + } + + tasks, err := b.api.DescribeTasks(ctx, cluster, arns...) + if err != nil { + return []types.TaskStatus{}, err + } + + networkInterfaces := []string{} + for _, t := range tasks { + if t.NetworkInterface != "" { + networkInterfaces = append(networkInterfaces, t.NetworkInterface) + } + } + publicIps, err := b.api.GetPublicIPs(ctx, networkInterfaces...) + if err != nil { + return []types.TaskStatus{}, err + } + + sort.Slice(tasks, func(i, j int) bool { + return strings.Compare(tasks[i].Service, tasks[j].Service) < 0 + }) + + for i, t := range tasks { + ports := []string{} + s, err := project.GetService(t.Service) + if err != nil { + return []types.TaskStatus{}, err + } + for _, p := range s.Ports { + ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", publicIps[t.NetworkInterface], p.Published, p.Target, p.Protocol)) + } + tasks[i].Name = s.Name + tasks[i].Ports = ports + } + return tasks, nil +} diff --git a/ecs/pkg/amazon/logs.go b/ecs/pkg/amazon/backend/logs.go similarity index 73% rename from ecs/pkg/amazon/logs.go rename to ecs/pkg/amazon/backend/logs.go index 683ecc26..17963529 100644 --- a/ecs/pkg/amazon/logs.go +++ b/ecs/pkg/amazon/backend/logs.go @@ -1,4 +1,4 @@ -package amazon +package backend import ( "context" @@ -11,8 +11,8 @@ import ( "github.com/docker/ecs-plugin/pkg/console" ) -func (c *client) ComposeLogs(ctx context.Context, projectName string) error { - err := c.api.GetLogs(ctx, projectName, &logConsumer{ +func (b *Backend) ComposeLogs(ctx context.Context, projectName string) error { + err := b.api.GetLogs(ctx, projectName, &logConsumer{ colors: map[string]console.ColorFunc{}, width: 0, }) @@ -26,11 +26,6 @@ func (c *client) ComposeLogs(ctx context.Context, projectName string) error { return nil } -type logConsumer struct { - colors map[string]console.ColorFunc - width int -} - func (l *logConsumer) Log(service, container, message string) { cf, ok := l.colors[service] if !ok { @@ -54,10 +49,7 @@ func (l *logConsumer) computeWidth() { l.width = width + 3 } -type LogConsumer interface { - Log(service, container, message string) -} - -type logsAPI interface { - GetLogs(ctx context.Context, name string, consumer LogConsumer) error +type logConsumer struct { + colors map[string]console.ColorFunc + width int } diff --git a/ecs/pkg/amazon/backend/secrets.go b/ecs/pkg/amazon/backend/secrets.go new file mode 100644 index 00000000..f2ae7c67 --- /dev/null +++ b/ecs/pkg/amazon/backend/secrets.go @@ -0,0 +1,23 @@ +package backend + +import ( + "context" + + "github.com/docker/ecs-plugin/pkg/amazon/types" +) + +func (b Backend) CreateSecret(ctx context.Context, secret types.Secret) (string, error) { + return b.api.CreateSecret(ctx, secret) +} + +func (b Backend) InspectSecret(ctx context.Context, id string) (types.Secret, error) { + return b.api.InspectSecret(ctx, id) +} + +func (b Backend) ListSecrets(ctx context.Context) ([]types.Secret, error) { + return b.api.ListSecrets(ctx) +} + +func (b Backend) DeleteSecret(ctx context.Context, id string, recover bool) error { + return b.api.DeleteSecret(ctx, id, recover) +} diff --git a/ecs/pkg/amazon/testdata/input/simple-single-service-with-overrides.yaml b/ecs/pkg/amazon/backend/testdata/input/simple-single-service-with-overrides.yaml similarity index 100% rename from ecs/pkg/amazon/testdata/input/simple-single-service-with-overrides.yaml rename to ecs/pkg/amazon/backend/testdata/input/simple-single-service-with-overrides.yaml diff --git a/ecs/pkg/amazon/testdata/input/simple-single-service.yaml b/ecs/pkg/amazon/backend/testdata/input/simple-single-service.yaml similarity index 100% rename from ecs/pkg/amazon/testdata/input/simple-single-service.yaml rename to ecs/pkg/amazon/backend/testdata/input/simple-single-service.yaml diff --git a/ecs/pkg/amazon/testdata/invalid_network_mode.yaml b/ecs/pkg/amazon/backend/testdata/invalid_network_mode.yaml similarity index 100% rename from ecs/pkg/amazon/testdata/invalid_network_mode.yaml rename to ecs/pkg/amazon/backend/testdata/invalid_network_mode.yaml diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden similarity index 100% rename from ecs/pkg/amazon/testdata/simple/simple-cloudformation-conversion.golden rename to ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden diff --git a/ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden similarity index 100% rename from ecs/pkg/amazon/testdata/simple/simple-cloudformation-with-overrides-conversion.golden rename to ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go new file mode 100644 index 00000000..8a7a895f --- /dev/null +++ b/ecs/pkg/amazon/backend/up.go @@ -0,0 +1,100 @@ +package backend + +import ( + "context" + "fmt" + + "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/docker/ecs-plugin/pkg/compose" +) + +func (b *Backend) ComposeUp(ctx context.Context, project *compose.Project) error { + if b.Cluster != "" { + ok, err := b.api.ClusterExists(ctx, b.Cluster) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("configured cluster %q does not exist", b.Cluster) + } + } + + update, err := b.api.StackExists(ctx, project.Name) + if err != nil { + return err + } + if update { + return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") + } + + template, err := b.Convert(project) + if err != nil { + return err + } + + vpc, err := b.GetVPC(ctx, project) + if err != nil { + return err + } + + subNets, err := b.api.GetSubNets(ctx, vpc) + if err != nil { + return err + } + + lb, err := b.GetLoadBalancer(ctx, project) + if err != nil { + return err + } + + parameters := map[string]string{ + ParameterClusterName: b.Cluster, + ParameterVPCId: vpc, + ParameterSubnet1Id: subNets[0], + ParameterSubnet2Id: subNets[1], + ParameterLoadBalancerARN: lb, + } + + err = b.api.CreateStack(ctx, project.Name, template, parameters) + if err != nil { + return err + } + + fmt.Println() + return b.WaitStackCompletion(ctx, project.Name, types.StackCreate) +} + +func (b Backend) GetVPC(ctx context.Context, project *compose.Project) (string, error) { + //check compose file for custom VPC selected + if vpc, ok := project.Extras[types.ExtensionVPC]; ok { + vpcID := vpc.(string) + ok, err := b.api.VpcExists(ctx, vpcID) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("VPC does not exist: %s", vpc) + } + } + defaultVPC, err := b.api.GetDefaultVPC(ctx) + if err != nil { + return "", err + } + return defaultVPC, nil +} + +func (b Backend) GetLoadBalancer(ctx context.Context, project *compose.Project) (string, error) { + //check compose file for custom VPC selected + if lb, ok := project.Extras[types.ExtensionLB]; ok { + lbName := lb.(string) + ok, err := b.api.LoadBalancerExists(ctx, lbName) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("Load Balancer does not exist: %s", lb) + } + return b.api.GetLoadBalancerARN(ctx, lbName) + } + return "", nil +} diff --git a/ecs/pkg/amazon/wait.go b/ecs/pkg/amazon/backend/wait.go similarity index 67% rename from ecs/pkg/amazon/wait.go rename to ecs/pkg/amazon/backend/wait.go index 58ae93d7..77ad844a 100644 --- a/ecs/pkg/amazon/wait.go +++ b/ecs/pkg/amazon/backend/wait.go @@ -1,4 +1,4 @@ -package amazon +package backend import ( "context" @@ -8,16 +8,15 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/docker/ecs-plugin/pkg/console" ) -func (c *client) WaitStackCompletion(ctx context.Context, name string, operation int) error { +func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int) error { w := console.NewProgressWriter() knownEvents := map[string]struct{}{} // Get the unique Stack ID so we can collect events without getting some from previous deployments with same name - stackID, err := c.api.GetStackID(ctx, name) + stackID, err := b.api.GetStackID(ctx, name) if err != nil { return err } @@ -26,7 +25,7 @@ func (c *client) WaitStackCompletion(ctx context.Context, name string, operation done := make(chan bool) go func() { - c.api.WaitStackComplete(ctx, stackID, operation) //nolint:errcheck + b.api.WaitStackComplete(ctx, stackID, operation) //nolint:errcheck ticker.Stop() done <- true }() @@ -39,7 +38,7 @@ func (c *client) WaitStackCompletion(ctx context.Context, name string, operation completed = true case <-ticker.C: } - events, err := c.api.DescribeStackEvents(ctx, stackID) + events, err := b.api.DescribeStackEvents(ctx, stackID) if err != nil { return err } @@ -65,14 +64,3 @@ func (c *client) WaitStackCompletion(ctx context.Context, name string, operation } return stackErr } - -type waitAPI interface { - GetStackID(ctx context.Context, name string) (string, error) - WaitStackComplete(ctx context.Context, name string, operation int) error - DescribeStackEvents(ctx context.Context, stackID string) ([]*cloudformation.StackEvent, error) -} - -const ( - StackCreate = iota - StackDelete -) diff --git a/ecs/pkg/amazon/check_test.go b/ecs/pkg/amazon/check_test.go deleted file mode 100644 index 3038eee8..00000000 --- a/ecs/pkg/amazon/check_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package amazon - -import ( - "testing" - - "gotest.tools/v3/assert" -) - -func TestInvalidNetworkMode(t *testing.T) { - project := load(t, "testdata/invalid_network_mode.yaml") - err := Check(project) - assert.Error(t, err[0], "'network_mode' \"bridge\" is not supported") -} diff --git a/ecs/pkg/amazon/check.go b/ecs/pkg/amazon/compatibility/check.go similarity index 98% rename from ecs/pkg/amazon/check.go rename to ecs/pkg/amazon/compatibility/check.go index 169794c7..58c4d306 100644 --- a/ecs/pkg/amazon/check.go +++ b/ecs/pkg/amazon/compatibility/check.go @@ -1,4 +1,4 @@ -package amazon +package compatibility import ( "github.com/compose-spec/compose-go/types" diff --git a/ecs/pkg/amazon/compatibility/check_test.go b/ecs/pkg/amazon/compatibility/check_test.go new file mode 100644 index 00000000..f6589853 --- /dev/null +++ b/ecs/pkg/amazon/compatibility/check_test.go @@ -0,0 +1,23 @@ +package compatibility + +import ( + "testing" + + "github.com/docker/ecs-plugin/pkg/compose" + "gotest.tools/v3/assert" +) + +func load(t *testing.T, paths ...string) *compose.Project { + options := compose.ProjectOptions{ + Name: t.Name(), + ConfigPaths: paths, + } + project, err := compose.ProjectFromOptions(&options) + assert.NilError(t, err) + return project +} +func TestInvalidNetworkMode(t *testing.T) { + project := load(t, "../backend/testdata/invalid_network_mode.yaml") + err := Check(project) + assert.Error(t, err[0], "'network_mode' \"bridge\" is not supported") +} diff --git a/ecs/pkg/amazon/compatibility.go b/ecs/pkg/amazon/compatibility/compatibility.go similarity index 99% rename from ecs/pkg/amazon/compatibility.go rename to ecs/pkg/amazon/compatibility/compatibility.go index 1a7f136a..a0fff142 100644 --- a/ecs/pkg/amazon/compatibility.go +++ b/ecs/pkg/amazon/compatibility/compatibility.go @@ -1,4 +1,4 @@ -package amazon +package compatibility import ( "fmt" diff --git a/ecs/pkg/amazon/down.go b/ecs/pkg/amazon/down.go deleted file mode 100644 index d9bee735..00000000 --- a/ecs/pkg/amazon/down.go +++ /dev/null @@ -1,34 +0,0 @@ -package amazon - -import ( - "context" - "fmt" -) - -func (c *client) ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error { - err := c.api.DeleteStack(ctx, projectName) - if err != nil { - return err - } - - err = c.WaitStackCompletion(ctx, projectName, StackDelete) - if err != nil { - return err - } - - if !deleteCluster { - return nil - } - - fmt.Printf("Delete cluster %s", c.Cluster) - if err = c.api.DeleteCluster(ctx, c.Cluster); err != nil { - return err - } - fmt.Printf("... done. \n") - return nil -} - -type downAPI interface { - DeleteStack(ctx context.Context, name string) error - DeleteCluster(ctx context.Context, name string) error -} diff --git a/ecs/pkg/amazon/list.go b/ecs/pkg/amazon/list.go deleted file mode 100644 index 98903f40..00000000 --- a/ecs/pkg/amazon/list.go +++ /dev/null @@ -1,80 +0,0 @@ -package amazon - -import ( - "context" - "fmt" - "os" - "sort" - "strings" - "text/tabwriter" - - "github.com/docker/ecs-plugin/pkg/compose" -) - -func (c *client) ComposePs(ctx context.Context, project *compose.Project) error { - cluster := c.Cluster - if cluster == "" { - cluster = project.Name - } - w := tabwriter.NewWriter(os.Stdout, 20, 2, 3, ' ', 0) - fmt.Fprintf(w, "Name\tState\tPorts\n") - defer w.Flush() - - arns := []string{} - for _, service := range project.Services { - tasks, err := c.api.ListTasks(ctx, cluster, service.Name) - if err != nil { - return err - } - arns = append(arns, tasks...) - } - if len(arns) == 0 { - return nil - } - - tasks, err := c.api.DescribeTasks(ctx, cluster, arns...) - if err != nil { - return err - } - - networkInterfaces := []string{} - for _, t := range tasks { - if t.NetworkInterface != "" { - networkInterfaces = append(networkInterfaces, t.NetworkInterface) - } - } - publicIps, err := c.api.GetPublicIPs(ctx, networkInterfaces...) - if err != nil { - return err - } - - sort.Slice(tasks, func(i, j int) bool { - return strings.Compare(tasks[i].Service, tasks[j].Service) < 0 - }) - - for _, t := range tasks { - ports := []string{} - s, err := project.GetService(t.Service) - if err != nil { - return err - } - for _, p := range s.Ports { - ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", publicIps[t.NetworkInterface], p.Published, p.Target, p.Protocol)) - } - fmt.Fprintf(w, "%s\t%s\t%s\n", s.Name, t.State, strings.Join(ports, ", ")) - } - return nil -} - -type TaskStatus struct { - State string - Service string - NetworkInterface string - PublicIP string -} - -type listAPI interface { - ListTasks(ctx context.Context, cluster string, name string) ([]string, error) - DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]TaskStatus, error) - GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error) -} diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go new file mode 100644 index 00000000..137a4339 --- /dev/null +++ b/ecs/pkg/amazon/sdk/api.go @@ -0,0 +1,61 @@ +package sdk + +import ( + "context" + + cf "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/awslabs/goformation/v4/cloudformation" + "github.com/docker/ecs-plugin/pkg/amazon/types" +) + +//go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon" -package=amazon . API + +type API interface { + downAPI + upAPI + logsAPI + secretsAPI + listAPI +} + +type upAPI interface { + waitAPI + GetDefaultVPC(ctx context.Context) (string, error) + VpcExists(ctx context.Context, vpcID string) (bool, error) + GetSubNets(ctx context.Context, vpcID string) ([]string, error) + + ClusterExists(ctx context.Context, name string) (bool, error) + StackExists(ctx context.Context, name string) (bool, error) + CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error + + LoadBalancerExists(ctx context.Context, name string) (bool, error) + GetLoadBalancerARN(ctx context.Context, name string) (string, error) +} + +type downAPI interface { + DeleteStack(ctx context.Context, name string) error + DeleteCluster(ctx context.Context, name string) error +} + +type logsAPI interface { + GetLogs(ctx context.Context, name string, consumer types.LogConsumer) error +} + +type secretsAPI interface { + CreateSecret(ctx context.Context, secret types.Secret) (string, error) + InspectSecret(ctx context.Context, id string) (types.Secret, error) + ListSecrets(ctx context.Context) ([]types.Secret, error) + DeleteSecret(ctx context.Context, id string, recover bool) error +} + +type listAPI interface { + ListTasks(ctx context.Context, cluster string, name string) ([]string, error) + DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]types.TaskStatus, error) + GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error) +} + +type waitAPI interface { + GetStackID(ctx context.Context, name string) (string, error) + WaitStackComplete(ctx context.Context, name string, operation int) error + DescribeStackEvents(ctx context.Context, stackID string) ([]*cf.StackEvent, error) +} diff --git a/ecs/pkg/amazon/api_mock.go b/ecs/pkg/amazon/sdk/api_mock.go similarity index 96% rename from ecs/pkg/amazon/api_mock.go rename to ecs/pkg/amazon/sdk/api_mock.go index cbf2b6d1..07ce79e7 100644 --- a/ecs/pkg/amazon/api_mock.go +++ b/ecs/pkg/amazon/sdk/api_mock.go @@ -2,7 +2,7 @@ // Source: github.com/docker/ecs-plugin/pkg/amazon (interfaces: API) // Package amazon is a generated GoMock package. -package amazon +package sdk import ( context "context" @@ -10,7 +10,7 @@ import ( cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" - docker "github.com/docker/ecs-plugin/pkg/docker" + btypes "github.com/docker/ecs-plugin/pkg/amazon/types" gomock "github.com/golang/mock/gomock" ) @@ -53,7 +53,7 @@ func (mr *MockAPIMockRecorder) ClusterExists(arg0, arg1 interface{}) *gomock.Cal } // CreateSecret mocks base method -func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 docker.Secret) (string, error) { +func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 btypes.Secret) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateSecret", arg0, arg1) ret0, _ := ret[0].(string) @@ -139,14 +139,14 @@ func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomo } // DescribeTasks mocks base method -func (m *MockAPI) DescribeTasks(arg0 context.Context, arg1 string, arg2 ...string) ([]TaskStatus, error) { +func (m *MockAPI) DescribeTasks(arg0 context.Context, arg1 string, arg2 ...string) ([]btypes.TaskStatus, error) { m.ctrl.T.Helper() varargs := []interface{}{arg0, arg1} for _, a := range arg2 { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "DescribeTasks", varargs...) - ret0, _ := ret[0].([]TaskStatus) + ret0, _ := ret[0].([]btypes.TaskStatus) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -174,7 +174,7 @@ func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call { } // GetLogs mocks base method -func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 LogConsumer) error { +func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 btypes.LogConsumer) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetLogs", arg0, arg1, arg2) ret0, _ := ret[0].(error) @@ -238,10 +238,10 @@ func (mr *MockAPIMockRecorder) GetSubNets(arg0, arg1 interface{}) *gomock.Call { } // InspectSecret mocks base method -func (m *MockAPI) InspectSecret(arg0 context.Context, arg1 string) (docker.Secret, error) { +func (m *MockAPI) InspectSecret(arg0 context.Context, arg1 string) (btypes.Secret, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InspectSecret", arg0, arg1) - ret0, _ := ret[0].(docker.Secret) + ret0, _ := ret[0].(btypes.Secret) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -253,10 +253,10 @@ func (mr *MockAPIMockRecorder) InspectSecret(arg0, arg1 interface{}) *gomock.Cal } // ListSecrets mocks base method -func (m *MockAPI) ListSecrets(arg0 context.Context) ([]docker.Secret, error) { +func (m *MockAPI) ListSecrets(arg0 context.Context) ([]btypes.Secret, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListSecrets", arg0) - ret0, _ := ret[0].([]docker.Secret) + ret0, _ := ret[0].([]btypes.Secret) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/ecs/pkg/amazon/convert.go b/ecs/pkg/amazon/sdk/convert.go similarity index 98% rename from ecs/pkg/amazon/convert.go rename to ecs/pkg/amazon/sdk/convert.go index 82047ddb..8874aa83 100644 --- a/ecs/pkg/amazon/convert.go +++ b/ecs/pkg/amazon/sdk/convert.go @@ -1,4 +1,4 @@ -package amazon +package sdk import ( "fmt" @@ -13,6 +13,7 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/compose-spec/compose-go/types" "github.com/docker/cli/opts" + t "github.com/docker/ecs-plugin/pkg/amazon/types" "github.com/docker/ecs-plugin/pkg/compose" ) @@ -318,7 +319,7 @@ func getImage(image string) string { func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name for key, value := range service.Extras { - if key == ExtensionPullCredentials { + if key == t.ExtensionPullCredentials { return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: value.(string)} } } diff --git a/ecs/pkg/amazon/sdk.go b/ecs/pkg/amazon/sdk/sdk.go similarity index 94% rename from ecs/pkg/amazon/sdk.go rename to ecs/pkg/amazon/sdk/sdk.go index e074fb11..1e1cb849 100644 --- a/ecs/pkg/amazon/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -1,4 +1,4 @@ -package amazon +package sdk import ( "context" @@ -25,7 +25,8 @@ import ( cf "github.com/awslabs/goformation/v4/cloudformation" "github.com/sirupsen/logrus" - "github.com/docker/ecs-plugin/pkg/docker" + "github.com/docker/ecs-plugin/pkg/amazon/types" + t "github.com/docker/ecs-plugin/pkg/amazon/types" ) type sdk struct { @@ -188,9 +189,9 @@ func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) StackName: aws.String(name), } switch operation { - case StackCreate: + case t.StackCreate: return s.CF.WaitUntilStackCreateCompleteWithContext(ctx, input) - case StackDelete: + case t.StackDelete: return s.CF.WaitUntilStackDeleteCompleteWithContext(ctx, input) default: return fmt.Errorf("internal error: unexpected stack operation %d", operation) @@ -235,7 +236,7 @@ func (s sdk) DeleteStack(ctx context.Context, name string) error { return err } -func (s sdk) CreateSecret(ctx context.Context, secret docker.Secret) (string, error) { +func (s sdk) CreateSecret(ctx context.Context, secret t.Secret) (string, error) { logrus.Debug("Create secret " + secret.Name) secretStr, err := secret.GetCredString() if err != nil { @@ -253,17 +254,17 @@ func (s sdk) CreateSecret(ctx context.Context, secret docker.Secret) (string, er return *response.ARN, nil } -func (s sdk) InspectSecret(ctx context.Context, id string) (docker.Secret, error) { +func (s sdk) InspectSecret(ctx context.Context, id string) (t.Secret, error) { logrus.Debug("Inspect secret " + id) response, err := s.SM.DescribeSecret(&secretsmanager.DescribeSecretInput{SecretId: &id}) if err != nil { - return docker.Secret{}, err + return t.Secret{}, err } labels := map[string]string{} for _, tag := range response.Tags { labels[*tag.Key] = *tag.Value } - secret := docker.Secret{ + secret := t.Secret{ ID: *response.ARN, Name: *response.Name, Labels: labels, @@ -274,14 +275,14 @@ func (s sdk) InspectSecret(ctx context.Context, id string) (docker.Secret, error return secret, nil } -func (s sdk) ListSecrets(ctx context.Context) ([]docker.Secret, error) { +func (s sdk) ListSecrets(ctx context.Context) ([]t.Secret, error) { logrus.Debug("List secrets ...") response, err := s.SM.ListSecrets(&secretsmanager.ListSecretsInput{}) if err != nil { - return []docker.Secret{}, err + return []t.Secret{}, err } - var secrets []docker.Secret + var secrets []t.Secret for _, sec := range response.SecretList { @@ -293,7 +294,7 @@ func (s sdk) ListSecrets(ctx context.Context) ([]docker.Secret, error) { if sec.Description != nil { description = *sec.Description } - secrets = append(secrets, docker.Secret{ + secrets = append(secrets, t.Secret{ ID: *sec.ARN, Name: *sec.Name, Labels: labels, @@ -310,7 +311,7 @@ func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { return err } -func (s sdk) GetLogs(ctx context.Context, name string, consumer LogConsumer) error { +func (s sdk) GetLogs(ctx context.Context, name string, consumer types.LogConsumer) error { logGroup := fmt.Sprintf("/docker-compose/%s", name) var startTime int64 for { @@ -356,7 +357,7 @@ func (s sdk) ListTasks(ctx context.Context, cluster string, service string) ([]s return arns, nil } -func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]TaskStatus, error) { +func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]t.TaskStatus, error) { tasks, err := s.ECS.DescribeTasksWithContext(ctx, &ecs.DescribeTasksInput{ Cluster: aws.String(cluster), Tasks: aws.StringSlice(arns), @@ -364,7 +365,7 @@ func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) if err != nil { return nil, err } - result := []TaskStatus{} + result := []t.TaskStatus{} for _, task := range tasks.Tasks { var networkInterface string for _, attachement := range task.Attachments { @@ -376,7 +377,7 @@ func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) } } } - result = append(result, TaskStatus{ + result = append(result, t.TaskStatus{ State: *task.LastStatus, Service: strings.Replace(*task.Group, "service:", "", 1), NetworkInterface: networkInterface, diff --git a/ecs/pkg/amazon/secrets.go b/ecs/pkg/amazon/secrets.go deleted file mode 100644 index 96a2a476..00000000 --- a/ecs/pkg/amazon/secrets.go +++ /dev/null @@ -1,30 +0,0 @@ -package amazon - -import ( - "context" - - "github.com/docker/ecs-plugin/pkg/docker" -) - -type secretsAPI interface { - CreateSecret(ctx context.Context, secret docker.Secret) (string, error) - InspectSecret(ctx context.Context, id string) (docker.Secret, error) - ListSecrets(ctx context.Context) ([]docker.Secret, error) - DeleteSecret(ctx context.Context, id string, recover bool) error -} - -func (c client) CreateSecret(ctx context.Context, secret docker.Secret) (string, error) { - return c.api.CreateSecret(ctx, secret) -} - -func (c client) InspectSecret(ctx context.Context, id string) (docker.Secret, error) { - return c.api.InspectSecret(ctx, id) -} - -func (c client) ListSecrets(ctx context.Context) ([]docker.Secret, error) { - return c.api.ListSecrets(ctx) -} - -func (c client) DeleteSecret(ctx context.Context, id string, recover bool) error { - return c.api.DeleteSecret(ctx, id, recover) -} diff --git a/ecs/pkg/docker/secret.go b/ecs/pkg/amazon/types/types.go similarity index 71% rename from ecs/pkg/docker/secret.go rename to ecs/pkg/amazon/types/types.go index 613c6263..f6815955 100644 --- a/ecs/pkg/docker/secret.go +++ b/ecs/pkg/amazon/types/types.go @@ -1,9 +1,25 @@ -package docker +package types -import ( - "encoding/json" +import "encoding/json" + +type TaskStatus struct { + Name string + State string + Service string + NetworkInterface string + PublicIP string + Ports []string +} + +const ( + StackCreate = iota + StackDelete ) +type LogConsumer interface { + Log(service, container, message string) +} + type Secret struct { ID string `json:"ID"` Name string `json:"Name"` diff --git a/ecs/pkg/amazon/x.go b/ecs/pkg/amazon/types/x.go similarity index 93% rename from ecs/pkg/amazon/x.go rename to ecs/pkg/amazon/types/x.go index 315c50c4..d8b8e011 100644 --- a/ecs/pkg/amazon/x.go +++ b/ecs/pkg/amazon/types/x.go @@ -1,4 +1,4 @@ -package amazon +package types const ( ExtensionSecurityGroup = "x-aws-securitygroup" diff --git a/ecs/pkg/amazon/up.go b/ecs/pkg/amazon/up.go deleted file mode 100644 index 3c947724..00000000 --- a/ecs/pkg/amazon/up.go +++ /dev/null @@ -1,114 +0,0 @@ -package amazon - -import ( - "context" - "fmt" - - "github.com/awslabs/goformation/v4/cloudformation" - "github.com/docker/ecs-plugin/pkg/compose" -) - -func (c *client) ComposeUp(ctx context.Context, project *compose.Project) error { - if c.Cluster != "" { - ok, err := c.api.ClusterExists(ctx, c.Cluster) - if err != nil { - return err - } - if !ok { - return fmt.Errorf("configured cluster %q does not exist", c.Cluster) - } - } - - update, err := c.api.StackExists(ctx, project.Name) - if err != nil { - return err - } - if update { - return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") - } - - template, err := c.Convert(project) - if err != nil { - return err - } - - vpc, err := c.GetVPC(ctx, project) - if err != nil { - return err - } - - subNets, err := c.api.GetSubNets(ctx, vpc) - if err != nil { - return err - } - - lb, err := c.GetLoadBalancer(ctx, project) - if err != nil { - return err - } - - parameters := map[string]string{ - ParameterClusterName: c.Cluster, - ParameterVPCId: vpc, - ParameterSubnet1Id: subNets[0], - ParameterSubnet2Id: subNets[1], - ParameterLoadBalancerARN: lb, - } - - err = c.api.CreateStack(ctx, project.Name, template, parameters) - if err != nil { - return err - } - - fmt.Println() - return c.WaitStackCompletion(ctx, project.Name, StackCreate) -} - -func (c client) GetVPC(ctx context.Context, project *compose.Project) (string, error) { - //check compose file for custom VPC selected - if vpc, ok := project.Extras[ExtensionVPC]; ok { - vpcID := vpc.(string) - ok, err := c.api.VpcExists(ctx, vpcID) - if err != nil { - return "", err - } - if !ok { - return "", fmt.Errorf("VPC does not exist: %s", vpc) - } - } - defaultVPC, err := c.api.GetDefaultVPC(ctx) - if err != nil { - return "", err - } - return defaultVPC, nil -} - -func (c client) GetLoadBalancer(ctx context.Context, project *compose.Project) (string, error) { - //check compose file for custom VPC selected - if lb, ok := project.Extras[ExtensionLB]; ok { - lbName := lb.(string) - ok, err := c.api.LoadBalancerExists(ctx, lbName) - if err != nil { - return "", err - } - if !ok { - return "", fmt.Errorf("Load Balancer does not exist: %s", lb) - } - return c.api.GetLoadBalancerARN(ctx, lbName) - } - return "", nil -} - -type upAPI interface { - waitAPI - GetDefaultVPC(ctx context.Context) (string, error) - VpcExists(ctx context.Context, vpcID string) (bool, error) - GetSubNets(ctx context.Context, vpcID string) ([]string, error) - - ClusterExists(ctx context.Context, name string) (bool, error) - StackExists(ctx context.Context, name string) (bool, error) - CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error - - LoadBalancerExists(ctx context.Context, name string) (bool, error) - GetLoadBalancerARN(ctx context.Context, name string) (string, error) -} diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 1049a993..016d4947 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -4,7 +4,7 @@ import ( "context" "github.com/awslabs/goformation/v4/cloudformation" - "github.com/docker/ecs-plugin/pkg/docker" + "github.com/docker/ecs-plugin/pkg/amazon/types" ) type API interface { @@ -13,9 +13,9 @@ type API interface { ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error ComposeLogs(ctx context.Context, projectName string) error - CreateSecret(ctx context.Context, secret docker.Secret) (string, error) - InspectSecret(ctx context.Context, id string) (docker.Secret, error) - ListSecrets(ctx context.Context) ([]docker.Secret, error) + CreateSecret(ctx context.Context, secret types.Secret) (string, error) + InspectSecret(ctx context.Context, id string) (types.Secret, error) + ListSecrets(ctx context.Context) ([]types.Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error - ComposePs(background context.Context, project *Project) error + ComposePs(background context.Context, project *Project) ([]types.TaskStatus, error) } From 9e8ddb63cc10427c7b553acd5e538047e2d2da0f Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 12 Jun 2020 15:00:27 +0200 Subject: [PATCH 125/205] fix lint issue - renamed CompatibilityChecker to Checker Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/compatibility/check.go | 2 +- ecs/pkg/amazon/compatibility/compatibility.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/pkg/amazon/compatibility/check.go b/ecs/pkg/amazon/compatibility/check.go index 58c4d306..8e46becb 100644 --- a/ecs/pkg/amazon/compatibility/check.go +++ b/ecs/pkg/amazon/compatibility/check.go @@ -8,7 +8,7 @@ import ( type Warning string type Warnings []string -type CompatibilityChecker interface { +type Checker interface { CheckService(service *types.ServiceConfig) CheckCapAdd(service *types.ServiceConfig) CheckDNS(service *types.ServiceConfig) diff --git a/ecs/pkg/amazon/compatibility/compatibility.go b/ecs/pkg/amazon/compatibility/compatibility.go index a0fff142..c7d67c26 100644 --- a/ecs/pkg/amazon/compatibility/compatibility.go +++ b/ecs/pkg/amazon/compatibility/compatibility.go @@ -168,4 +168,4 @@ func (c *FargateCompatibilityChecker) CheckLabels(service *types.ServiceConfig) } } -var _ CompatibilityChecker = &FargateCompatibilityChecker{} +var _ Checker = &FargateCompatibilityChecker{} From 1bb95134f0c0a846be4201d533fd9c8995cf7783 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 15 Jun 2020 10:57:51 +0200 Subject: [PATCH 126/205] match docker/api signature for up and down methods Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 28 +++++----------------------- ecs/pkg/amazon/backend/down.go | 17 ++++++----------- ecs/pkg/amazon/backend/down_test.go | 29 ++++++----------------------- ecs/pkg/amazon/backend/list.go | 2 +- ecs/pkg/amazon/backend/logs.go | 2 +- ecs/pkg/amazon/backend/up.go | 7 ++++++- ecs/pkg/compose/api.go | 9 +++++---- 7 files changed, 30 insertions(+), 64 deletions(-) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 55eb7a14..fc5802e0 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -75,16 +75,12 @@ func UpCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobr opts := upOptions{} cmd := &cobra.Command{ Use: "up", - RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { - clusteropts, err := docker.GetAwsContext(dockerCli) - if err != nil { - return err - } + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) if err != nil { return err } - return backend.ComposeUp(context.Background(), project) + return backend.Up(context.Background(), *projectOpts) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") @@ -104,7 +100,7 @@ func PsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobr if err != nil { return err } - tasks, err := backend.ComposePs(context.Background(), project) + tasks, err := backend.Ps(context.Background(), project) if err != nil { return err } @@ -133,21 +129,7 @@ func DownCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co if err != nil { return err } - if len(args) == 0 { - project, err := compose.ProjectFromOptions(projectOpts) - if err != nil { - return err - } - return backend.ComposeDown(context.Background(), project.Name, opts.DeleteCluster) - } - // project names passed as parameters - for _, name := range args { - err := backend.ComposeDown(context.Background(), name, opts.DeleteCluster) - if err != nil { - return err - } - } - return nil + return backend.Down(context.Background(), *projectOpts) }), } cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") @@ -173,7 +155,7 @@ func LogsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co } else { name = args[0] } - return backend.ComposeLogs(context.Background(), name) + return backend.Logs(context.Background(), name) }), } return cmd diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index 8b978e79..d07c4a06 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -2,30 +2,25 @@ package backend import ( "context" - "fmt" "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error { - err := b.api.DeleteStack(ctx, projectName) +func (b *Backend) Down(ctx context.Context, options compose.ProjectOptions) error { + project, err := compose.ProjectFromOptions(&options) if err != nil { return err } - err = b.WaitStackCompletion(ctx, projectName, types.StackDelete) + err = b.api.DeleteStack(ctx, project.Name) if err != nil { return err } - if !deleteCluster { - return nil - } - - fmt.Printf("Delete cluster %s", b.Cluster) - if err = b.api.DeleteCluster(ctx, b.Cluster); err != nil { + err = b.WaitStackCompletion(ctx, project.Name, types.StackDelete) + if err != nil { return err } - fmt.Printf("... done. \n") return nil } diff --git a/ecs/pkg/amazon/backend/down_test.go b/ecs/pkg/amazon/backend/down_test.go index d9abf70c..7d156e24 100644 --- a/ecs/pkg/amazon/backend/down_test.go +++ b/ecs/pkg/amazon/backend/down_test.go @@ -6,10 +6,11 @@ import ( "github.com/docker/ecs-plugin/pkg/amazon/sdk" btypes "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/docker/ecs-plugin/pkg/compose" "github.com/golang/mock/gomock" ) -func TestDownDontDeleteCluster(t *testing.T) { +func TestDown(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := sdk.NewMockAPI(ctrl) @@ -25,26 +26,8 @@ func TestDownDontDeleteCluster(t *testing.T) { recorder.WaitStackComplete(ctx, "stack-123", btypes.StackDelete).Return(nil) recorder.DescribeStackEvents(ctx, "stack-123").Return(nil, nil) - c.ComposeDown(ctx, "test_project", false) -} - -func TestDownDeleteCluster(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - m := sdk.NewMockAPI(ctrl) - c := &Backend{ - Cluster: "test_cluster", - Region: "region", - api: m, - } - - ctx := context.TODO() - recorder := m.EXPECT() - recorder.DeleteStack(ctx, "test_project").Return(nil) - recorder.GetStackID(ctx, "test_project").Return("stack-123", nil) - recorder.WaitStackComplete(ctx, "stack-123", btypes.StackDelete).Return(nil) - recorder.DescribeStackEvents(ctx, "stack-123").Return(nil, nil) - recorder.DeleteCluster(ctx, "test_cluster").Return(nil) - - c.ComposeDown(ctx, "test_project", true) + c.Down(ctx, compose.ProjectOptions{ + ConfigPaths: []string{}, + Name: "test_project", + }) } diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index 055362aa..b9090594 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -10,7 +10,7 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) ComposePs(ctx context.Context, project *compose.Project) ([]types.TaskStatus, error) { +func (b *Backend) Ps(ctx context.Context, project *compose.Project) ([]types.TaskStatus, error) { cluster := b.Cluster if cluster == "" { cluster = project.Name diff --git a/ecs/pkg/amazon/backend/logs.go b/ecs/pkg/amazon/backend/logs.go index 17963529..a1715885 100644 --- a/ecs/pkg/amazon/backend/logs.go +++ b/ecs/pkg/amazon/backend/logs.go @@ -11,7 +11,7 @@ import ( "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) ComposeLogs(ctx context.Context, projectName string) error { +func (b *Backend) Logs(ctx context.Context, projectName string) error { err := b.api.GetLogs(ctx, projectName, &logConsumer{ colors: map[string]console.ColorFunc{}, width: 0, diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 8a7a895f..8c78748a 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -8,7 +8,12 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) ComposeUp(ctx context.Context, project *compose.Project) error { +func (b *Backend) Up(ctx context.Context, options compose.ProjectOptions) error { + project, err := compose.ProjectFromOptions(&options) + if err != nil { + return err + } + if b.Cluster != "" { ok, err := b.api.ClusterExists(ctx, b.Cluster) if err != nil { diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 016d4947..6d84ccec 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -8,14 +8,15 @@ import ( ) type API interface { + Up(ctx context.Context, options ProjectOptions) error + Down(ctx context.Context, options ProjectOptions) error + Convert(project *Project) (*cloudformation.Template, error) - ComposeUp(ctx context.Context, project *Project) error - ComposeDown(ctx context.Context, projectName string, deleteCluster bool) error - ComposeLogs(ctx context.Context, projectName string) error + Logs(ctx context.Context, projectName string) error + Ps(background context.Context, project *Project) ([]types.TaskStatus, error) CreateSecret(ctx context.Context, secret types.Secret) (string, error) InspectSecret(ctx context.Context, id string) (types.Secret, error) ListSecrets(ctx context.Context) ([]types.Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error - ComposePs(background context.Context, project *Project) ([]types.TaskStatus, error) } From dcf84f24998a756a1bd6a4b2930b8da76f314d77 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 15 Jun 2020 11:25:52 +0200 Subject: [PATCH 127/205] Fix broken build on master Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/down.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index d07c4a06..adbf9af3 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -8,17 +8,21 @@ import ( ) func (b *Backend) Down(ctx context.Context, options compose.ProjectOptions) error { - project, err := compose.ProjectFromOptions(&options) + name := options.Name + if name == "" { + project, err := compose.ProjectFromOptions(&options) + if err != nil { + return err + } + name = project.Name + } + + err := b.api.DeleteStack(ctx, name) if err != nil { return err } - err = b.api.DeleteStack(ctx, project.Name) - if err != nil { - return err - } - - err = b.WaitStackCompletion(ctx, project.Name, types.StackDelete) + err = b.WaitStackCompletion(ctx, name, types.StackDelete) if err != nil { return err } From c5895fe09af30eb1336877211c7833b7ebc18cbd Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 17 Jun 2020 14:48:42 +0200 Subject: [PATCH 128/205] Use `Project` from compose-go Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 23 +-- ecs/{pkg/compose => cmd/commands}/opts.go | 17 +- ecs/cmd/commands/secret.go | 6 +- ecs/go.mod | 7 +- ecs/go.sum | 29 ++- ecs/pkg/amazon/backend/cloudformation.go | 65 ++++--- ecs/pkg/amazon/backend/cloudformation_test.go | 27 ++- ecs/pkg/amazon/backend/down.go | 8 +- ecs/pkg/amazon/backend/down_test.go | 6 +- ecs/pkg/amazon/backend/list.go | 20 +- ecs/pkg/amazon/backend/secrets.go | 8 +- ecs/pkg/amazon/backend/up.go | 17 +- ecs/pkg/amazon/compatibility/check.go | 41 ----- ecs/pkg/amazon/compatibility/check_test.go | 23 --- ecs/pkg/amazon/compatibility/compatibility.go | 171 ------------------ ecs/pkg/amazon/sdk/api.go | 12 +- ecs/pkg/amazon/sdk/api_mock.go | 19 +- ecs/pkg/amazon/sdk/convert.go | 7 +- ecs/pkg/amazon/sdk/sdk.go | 32 ++-- ecs/pkg/compose/api.go | 17 +- ecs/pkg/compose/normalize.go | 89 --------- ecs/pkg/compose/project.go | 170 ----------------- ecs/pkg/compose/project_test.go | 46 ----- ecs/pkg/{amazon/types => compose}/types.go | 2 +- ecs/pkg/{amazon/types => compose}/x.go | 2 +- 25 files changed, 178 insertions(+), 686 deletions(-) rename ecs/{pkg/compose => cmd/commands}/opts.go (53%) delete mode 100644 ecs/pkg/amazon/compatibility/check.go delete mode 100644 ecs/pkg/amazon/compatibility/check_test.go delete mode 100644 ecs/pkg/amazon/compatibility/compatibility.go delete mode 100644 ecs/pkg/compose/normalize.go delete mode 100644 ecs/pkg/compose/project.go delete mode 100644 ecs/pkg/compose/project_test.go rename ecs/pkg/{amazon/types => compose}/types.go (98%) rename ecs/pkg/{amazon/types => compose}/x.go (92%) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index fc5802e0..b3489bae 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -7,9 +7,10 @@ import ( "os" "strings" + "github.com/compose-spec/compose-go/cli" + "github.com/compose-spec/compose-go/types" "github.com/docker/cli/cli/command" amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" - "github.com/docker/ecs-plugin/pkg/compose" "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" ) @@ -18,8 +19,8 @@ func ComposeCommand(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "compose", } - opts := &compose.ProjectOptions{} - opts.AddFlags(cmd.Flags()) + opts := &cli.ProjectOptions{} + AddFlags(opts, cmd.Flags()) cmd.AddCommand( ConvertCommand(dockerCli, opts), @@ -42,10 +43,10 @@ func (o upOptions) LoadBalancerArn() *string { return &o.loadBalancerArn } -func ConvertCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { +func ConvertCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "convert", - RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + RunE: WithProject(projectOpts, func(project *types.Project, args []string) error { clusteropts, err := docker.GetAwsContext(dockerCli) if err != nil { return err @@ -71,7 +72,7 @@ func ConvertCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) return cmd } -func UpCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { +func UpCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ Use: "up", @@ -87,11 +88,11 @@ func UpCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobr return cmd } -func PsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { +func PsCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ Use: "ps", - RunE: compose.WithProject(projectOpts, func(project *compose.Project, args []string) error { + RunE: WithProject(projectOpts, func(project *types.Project, args []string) error { clusteropts, err := docker.GetAwsContext(dockerCli) if err != nil { return err @@ -120,7 +121,7 @@ type downOptions struct { DeleteCluster bool } -func DownCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { +func DownCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { opts := downOptions{} cmd := &cobra.Command{ Use: "down", @@ -136,7 +137,7 @@ func DownCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co return cmd } -func LogsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *cobra.Command { +func LogsCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "logs [PROJECT NAME]", RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { @@ -147,7 +148,7 @@ func LogsCommand(dockerCli command.Cli, projectOpts *compose.ProjectOptions) *co var name string if len(args) == 0 { - project, err := compose.ProjectFromOptions(projectOpts) + project, err := cli.ProjectFromOptions(projectOpts) if err != nil { return err } diff --git a/ecs/pkg/compose/opts.go b/ecs/cmd/commands/opts.go similarity index 53% rename from ecs/pkg/compose/opts.go rename to ecs/cmd/commands/opts.go index d2fef0c9..7d0856c8 100644 --- a/ecs/pkg/compose/opts.go +++ b/ecs/cmd/commands/opts.go @@ -1,26 +1,23 @@ -package compose +package commands import ( + "github.com/compose-spec/compose-go/cli" + "github.com/compose-spec/compose-go/types" "github.com/spf13/cobra" "github.com/spf13/pflag" ) -type ProjectOptions struct { - ConfigPaths []string - Name string -} - -func (o *ProjectOptions) AddFlags(flags *pflag.FlagSet) { +func AddFlags(o *cli.ProjectOptions, flags *pflag.FlagSet) { flags.StringArrayVarP(&o.ConfigPaths, "file", "f", nil, "Specify an alternate compose file") flags.StringVarP(&o.Name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") } -type ProjectFunc func(project *Project, args []string) error +type ProjectFunc func(project *types.Project, args []string) error // WithProject wrap a ProjectFunc into a cobra command -func WithProject(options *ProjectOptions, f ProjectFunc) func(cmd *cobra.Command, args []string) error { +func WithProject(options *cli.ProjectOptions, f ProjectFunc) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { - project, err := ProjectFromOptions(options) + project, err := cli.ProjectFromOptions(options) if err != nil { return err } diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go index a426740d..b6a32c78 100644 --- a/ecs/cmd/commands/secret.go +++ b/ecs/cmd/commands/secret.go @@ -11,7 +11,7 @@ import ( "github.com/docker/cli/cli/command" amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" - "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/docker/ecs-plugin/pkg/compose" "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" ) @@ -57,7 +57,7 @@ func CreateSecret(dockerCli command.Cli) *cobra.Command { } name := args[0] - secret := types.NewSecret(name, opts.Username, opts.Password, opts.Description) + secret := compose.NewSecret(name, opts.Username, opts.Password, opts.Description) id, err := backend.CreateSecret(context.Background(), secret) fmt.Println(id) return err @@ -140,7 +140,7 @@ func DeleteSecret(dockerCli command.Cli) *cobra.Command { return cmd } -func printList(out io.Writer, secrets []types.Secret) { +func printList(out io.Writer, secrets []compose.Secret) { printSection(out, len(secrets), func(w io.Writer) { for _, secret := range secrets { fmt.Fprintf(w, "%s\t%s\t%s\n", secret.ID, secret.Name, secret.Description) diff --git a/ecs/go.mod b/ecs/go.mod index 7dd8d7fe..438cf8b0 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -14,7 +14,7 @@ require ( github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200409090215-53c0040c9127 + github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 @@ -35,11 +35,11 @@ require ( github.com/manifoldco/promptui v0.7.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/miekg/pkcs11 v1.0.3 // indirect - github.com/mitchellh/mapstructure v1.2.2 + github.com/mitchellh/mapstructure v1.3.2 github.com/morikuni/aec v1.0.0 // indirect github.com/onsi/ginkgo v1.11.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/sirupsen/logrus v1.5.0 + github.com/sirupsen/logrus v1.6.0 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 @@ -52,7 +52,6 @@ require ( gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect gopkg.in/ini.v1 v1.55.0 - gotest.tools v2.2.0+incompatible gotest.tools/v3 v3.0.2 vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect ) diff --git a/ecs/go.sum b/ecs/go.sum index f0303a30..9edabc47 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -54,8 +54,12 @@ github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiK github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo= github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= -github.com/compose-spec/compose-go v0.0.0-20200409090215-53c0040c9127 h1:mAsQN3s19glh3KBOQjiRYBhqaX1SdzNqhB3/cuqgSbE= -github.com/compose-spec/compose-go v0.0.0-20200409090215-53c0040c9127/go.mod h1:1PUpzRF1O/65VOqXZuwpCuYY7pJxbIq1jbAvAf62FGM= +github.com/compose-spec/compose-go v0.0.0-20200616184722-5b8dc203fd7f h1:XE6hHZdPjxN8uGaRlvdCB8YwXbz1PXnQ0CboNygdL2o= +github.com/compose-spec/compose-go v0.0.0-20200616184722-5b8dc203fd7f/go.mod h1:d3Vb4tH01Pr4YKD3RvfwguRcezDBUYJTVYgpCSRYSVg= +github.com/compose-spec/compose-go v0.0.0-20200617133919-fca3bb55c5cc h1:jZfF+HzxW+c8Em308MvcK7j5+ZqIAWqFjN1RZnVFzck= +github.com/compose-spec/compose-go v0.0.0-20200617133919-fca3bb55c5cc/go.mod h1:d3Vb4tH01Pr4YKD3RvfwguRcezDBUYJTVYgpCSRYSVg= +github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a h1:FmEuebUePUA0Kd/NSiCmdPG/n6eKdZdBtIbfejVtRS8= +github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -134,9 +138,12 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE= github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -151,6 +158,7 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -183,6 +191,8 @@ github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCy github.com/kisom/goutils v1.1.0/go.mod h1:+UBTfd78habUYWFbNWTJNG+jNG/i/lGURakr4A/yNRw= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -218,8 +228,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= -github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -282,8 +292,8 @@ github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvH github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= -github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -330,6 +340,7 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56 h1:yhqBHs09SmmUoNOHc9jgK4a60T3XFRtPAkYxVnqgY50= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -450,6 +461,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 115a67d6..5cdbcdc2 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -5,14 +5,9 @@ import ( "regexp" "strings" - "github.com/compose-spec/compose-go/types" - - "github.com/sirupsen/logrus" - + ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/elbv2" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" - - ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" "github.com/awslabs/goformation/v4/cloudformation/ecs" @@ -21,10 +16,11 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/logs" cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" "github.com/awslabs/goformation/v4/cloudformation/tags" - "github.com/docker/ecs-plugin/pkg/amazon/compatibility" + "github.com/compose-spec/compose-go/compatibility" + "github.com/compose-spec/compose-go/types" sdk "github.com/docker/ecs-plugin/pkg/amazon/sdk" - btypes "github.com/docker/ecs-plugin/pkg/amazon/types" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/sirupsen/logrus" ) const ( @@ -35,11 +31,38 @@ const ( ParameterLoadBalancerARN = "ParameterLoadBalancerARN" ) +type FargateCompatibilityChecker struct { + *compatibility.AllowList +} + // Convert a compose project into a CloudFormation template -func (b Backend) Convert(project *compose.Project) (*cloudformation.Template, error) { - warnings := compatibility.Check(project) - for _, w := range warnings { - logrus.Warn(w) +func (b Backend) Convert(project *types.Project) (*cloudformation.Template, error) { + var checker compatibility.Checker = FargateCompatibilityChecker{ + &compatibility.AllowList{ + Supported: []string{ + "services.command", + "services.container_name", + "services.depends_on", + "services.entrypoint", + "services.environment", + "services.healthcheck", + "services.healthcheck.interval", + "services.healthcheck.start_period", + "services.healthcheck.test", + "services.healthcheck.timeout", + "services.networks", + "services.ports", + "services.ports.mode", + "services.ports.target", + "services.ports.protocol", + "services.user", + "services.working_dir", + }, + }, + } + compatibility.Check(project, checker) + for _, err := range checker.Errors() { + logrus.Warn(err.Error()) } template := cloudformation.NewTemplate() @@ -188,7 +211,7 @@ func (b Backend) Convert(project *compose.Project) (*cloudformation.Template, er return template, nil } -func getLoadBalancerType(project *compose.Project) string { +func getLoadBalancerType(project *types.Project) string { for _, service := range project.Services { for _, port := range service.Ports { if port.Published != 80 && port.Published != 443 { @@ -199,7 +222,7 @@ func getLoadBalancerType(project *compose.Project) string { return elbv2.LoadBalancerTypeEnumApplication } -func getLoadBalancerSecurityGroups(project *compose.Project, template *cloudformation.Template) []string { +func getLoadBalancerSecurityGroups(project *types.Project, template *cloudformation.Template) []string { securityGroups := []string{} for _, network := range project.Networks { if !network.Internal { @@ -210,7 +233,7 @@ func getLoadBalancerSecurityGroups(project *compose.Project, template *cloudform return uniqueStrings(securityGroups) } -func createLoadBalancer(project *compose.Project, template *cloudformation.Template) string { +func createLoadBalancer(project *types.Project, template *cloudformation.Template) string { loadBalancerName := fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name)) // Create LoadBalancer if `ParameterLoadBalancerName` is not set template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) @@ -270,7 +293,7 @@ func createListener(service types.ServiceConfig, port types.ServicePortConfig, t return listenerName } -func createTargetGroup(project *compose.Project, service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, protocol string) string { +func createTargetGroup(project *types.Project, service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, protocol string) string { targetGroupName := fmt.Sprintf( "%s%s%dTargetGroup", normalizeResourceName(service.Name), @@ -345,7 +368,7 @@ func createTaskExecutionRole(service types.ServiceConfig, err error, definition return taskExecutionRole, nil } -func createCluster(project *compose.Project, template *cloudformation.Template) string { +func createCluster(project *types.Project, template *cloudformation.Template) string { template.Resources["Cluster"] = &ecs.Cluster{ ClusterName: project.Name, Tags: []tags.Tag{ @@ -360,7 +383,7 @@ func createCluster(project *compose.Project, template *cloudformation.Template) return cluster } -func createCloudMap(project *compose.Project, template *cloudformation.Template) { +func createCloudMap(project *types.Project, template *cloudformation.Template) { template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), Name: fmt.Sprintf("%s.local", project.Name), @@ -368,8 +391,8 @@ func createCloudMap(project *compose.Project, template *cloudformation.Template) } } -func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc string, template *cloudformation.Template) string { - if sg, ok := net.Extras[btypes.ExtensionSecurityGroup]; ok { +func convertNetwork(project *types.Project, net types.NetworkConfig, vpc string, template *cloudformation.Template) string { + if sg, ok := net.Extensions[compose.ExtensionSecurityGroup]; ok { logrus.Debugf("Security Group for network %q set by user to %q", net.Name, sg) return sg.(string) } @@ -420,7 +443,7 @@ func convertNetwork(project *compose.Project, net types.NetworkConfig, vpc strin return cloudformation.Ref(securityGroup) } -func networkResourceName(project *compose.Project, network string) string { +func networkResourceName(project *types.Project, network string) string { return fmt.Sprintf("%s%sNetwork", normalizeResourceName(project.Name), normalizeResourceName(network)) } diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 12271478..001931b0 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -7,13 +7,11 @@ import ( "github.com/aws/aws-sdk-go/service/elbv2" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" - "github.com/awslabs/goformation/v4/cloudformation/iam" - "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" + "github.com/awslabs/goformation/v4/cloudformation/iam" + "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" - "github.com/docker/ecs-plugin/pkg/compose" - "gotest.tools/v3/assert" "gotest.tools/v3/golden" ) @@ -58,6 +56,10 @@ version: "3" services: test: image: hello_world + networks: + - front-tier + - back-tier + networks: front-tier: name: public @@ -103,7 +105,7 @@ services: assert.Check(t, lb.Type == elbv2.LoadBalancerTypeEnumNetwork) } -func convertResultAsString(t *testing.T, project *compose.Project, clusterName string) string { +func convertResultAsString(t *testing.T, project *types.Project, clusterName string) string { client, err := NewBackend("", clusterName, "") assert.NilError(t, err) result, err := client.Convert(project) @@ -113,12 +115,12 @@ func convertResultAsString(t *testing.T, project *compose.Project, clusterName s return fmt.Sprintf("%s\n", string(resultAsJSON)) } -func load(t *testing.T, paths ...string) *compose.Project { - options := compose.ProjectOptions{ +func load(t *testing.T, paths ...string) *types.Project { + options := cli.ProjectOptions{ Name: t.Name(), ConfigPaths: paths, } - project, err := compose.ProjectFromOptions(&options) + project, err := cli.ProjectFromOptions(&options) assert.NilError(t, err) return project } @@ -130,14 +132,11 @@ func convertYaml(t *testing.T, yaml string) *cloudformation.Template { ConfigFiles: []types.ConfigFile{ {Config: dict}, }, + }, func(options *loader.Options) { + options.Name = "Test" }) assert.NilError(t, err) - err = compose.Normalize(model) - assert.NilError(t, err) - template, err := Backend{}.Convert(&compose.Project{ - Config: *model, - Name: "test", - }) + template, err := Backend{}.Convert(model) assert.NilError(t, err) return template } diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index adbf9af3..18b4cbf9 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -3,14 +3,14 @@ package backend import ( "context" - "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/compose-spec/compose-go/cli" "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) Down(ctx context.Context, options compose.ProjectOptions) error { +func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { name := options.Name if name == "" { - project, err := compose.ProjectFromOptions(&options) + project, err := cli.ProjectFromOptions(&options) if err != nil { return err } @@ -22,7 +22,7 @@ func (b *Backend) Down(ctx context.Context, options compose.ProjectOptions) erro return err } - err = b.WaitStackCompletion(ctx, name, types.StackDelete) + err = b.WaitStackCompletion(ctx, name, compose.StackDelete) if err != nil { return err } diff --git a/ecs/pkg/amazon/backend/down_test.go b/ecs/pkg/amazon/backend/down_test.go index 7d156e24..7439f626 100644 --- a/ecs/pkg/amazon/backend/down_test.go +++ b/ecs/pkg/amazon/backend/down_test.go @@ -4,8 +4,8 @@ import ( "context" "testing" + "github.com/compose-spec/compose-go/cli" "github.com/docker/ecs-plugin/pkg/amazon/sdk" - btypes "github.com/docker/ecs-plugin/pkg/amazon/types" "github.com/docker/ecs-plugin/pkg/compose" "github.com/golang/mock/gomock" ) @@ -23,10 +23,10 @@ func TestDown(t *testing.T) { recorder := m.EXPECT() recorder.DeleteStack(ctx, "test_project").Return(nil) recorder.GetStackID(ctx, "test_project").Return("stack-123", nil) - recorder.WaitStackComplete(ctx, "stack-123", btypes.StackDelete).Return(nil) + recorder.WaitStackComplete(ctx, "stack-123", compose.StackDelete).Return(nil) recorder.DescribeStackEvents(ctx, "stack-123").Return(nil, nil) - c.Down(ctx, compose.ProjectOptions{ + c.Down(ctx, cli.ProjectOptions{ ConfigPaths: []string{}, Name: "test_project", }) diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index b9090594..4385eae5 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -6,11 +6,11 @@ import ( "sort" "strings" - "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) Ps(ctx context.Context, project *compose.Project) ([]types.TaskStatus, error) { +func (b *Backend) Ps(ctx context.Context, project *types.Project) ([]compose.TaskStatus, error) { cluster := b.Cluster if cluster == "" { cluster = project.Name @@ -19,17 +19,17 @@ func (b *Backend) Ps(ctx context.Context, project *compose.Project) ([]types.Tas for _, service := range project.Services { tasks, err := b.api.ListTasks(ctx, cluster, service.Name) if err != nil { - return []types.TaskStatus{}, err + return []compose.TaskStatus{}, err } arns = append(arns, tasks...) } if len(arns) == 0 { - return []types.TaskStatus{}, nil + return []compose.TaskStatus{}, nil } tasks, err := b.api.DescribeTasks(ctx, cluster, arns...) if err != nil { - return []types.TaskStatus{}, err + return []compose.TaskStatus{}, err } networkInterfaces := []string{} @@ -40,21 +40,21 @@ func (b *Backend) Ps(ctx context.Context, project *compose.Project) ([]types.Tas } publicIps, err := b.api.GetPublicIPs(ctx, networkInterfaces...) if err != nil { - return []types.TaskStatus{}, err + return []compose.TaskStatus{}, err } sort.Slice(tasks, func(i, j int) bool { return strings.Compare(tasks[i].Service, tasks[j].Service) < 0 }) - for i, t := range tasks { + for i, task := range tasks { ports := []string{} - s, err := project.GetService(t.Service) + s, err := project.GetService(task.Service) if err != nil { - return []types.TaskStatus{}, err + return []compose.TaskStatus{}, err } for _, p := range s.Ports { - ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", publicIps[t.NetworkInterface], p.Published, p.Target, p.Protocol)) + ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", publicIps[task.NetworkInterface], p.Published, p.Target, p.Protocol)) } tasks[i].Name = s.Name tasks[i].Ports = ports diff --git a/ecs/pkg/amazon/backend/secrets.go b/ecs/pkg/amazon/backend/secrets.go index f2ae7c67..6c86e95d 100644 --- a/ecs/pkg/amazon/backend/secrets.go +++ b/ecs/pkg/amazon/backend/secrets.go @@ -3,18 +3,18 @@ package backend import ( "context" - "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/docker/ecs-plugin/pkg/compose" ) -func (b Backend) CreateSecret(ctx context.Context, secret types.Secret) (string, error) { +func (b Backend) CreateSecret(ctx context.Context, secret compose.Secret) (string, error) { return b.api.CreateSecret(ctx, secret) } -func (b Backend) InspectSecret(ctx context.Context, id string) (types.Secret, error) { +func (b Backend) InspectSecret(ctx context.Context, id string) (compose.Secret, error) { return b.api.InspectSecret(ctx, id) } -func (b Backend) ListSecrets(ctx context.Context) ([]types.Secret, error) { +func (b Backend) ListSecrets(ctx context.Context) ([]compose.Secret, error) { return b.api.ListSecrets(ctx) } diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 8c78748a..24c75d5c 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -4,12 +4,13 @@ import ( "context" "fmt" - "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/compose-spec/compose-go/cli" + "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) Up(ctx context.Context, options compose.ProjectOptions) error { - project, err := compose.ProjectFromOptions(&options) +func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { + project, err := cli.ProjectFromOptions(&options) if err != nil { return err } @@ -66,12 +67,12 @@ func (b *Backend) Up(ctx context.Context, options compose.ProjectOptions) error } fmt.Println() - return b.WaitStackCompletion(ctx, project.Name, types.StackCreate) + return b.WaitStackCompletion(ctx, project.Name, compose.StackCreate) } -func (b Backend) GetVPC(ctx context.Context, project *compose.Project) (string, error) { +func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if vpc, ok := project.Extras[types.ExtensionVPC]; ok { + if vpc, ok := project.Extensions[compose.ExtensionVPC]; ok { vpcID := vpc.(string) ok, err := b.api.VpcExists(ctx, vpcID) if err != nil { @@ -88,9 +89,9 @@ func (b Backend) GetVPC(ctx context.Context, project *compose.Project) (string, return defaultVPC, nil } -func (b Backend) GetLoadBalancer(ctx context.Context, project *compose.Project) (string, error) { +func (b Backend) GetLoadBalancer(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if lb, ok := project.Extras[types.ExtensionLB]; ok { + if lb, ok := project.Extensions[compose.ExtensionLB]; ok { lbName := lb.(string) ok, err := b.api.LoadBalancerExists(ctx, lbName) if err != nil { diff --git a/ecs/pkg/amazon/compatibility/check.go b/ecs/pkg/amazon/compatibility/check.go deleted file mode 100644 index 8e46becb..00000000 --- a/ecs/pkg/amazon/compatibility/check.go +++ /dev/null @@ -1,41 +0,0 @@ -package compatibility - -import ( - "github.com/compose-spec/compose-go/types" - "github.com/docker/ecs-plugin/pkg/compose" -) - -type Warning string -type Warnings []string - -type Checker interface { - CheckService(service *types.ServiceConfig) - CheckCapAdd(service *types.ServiceConfig) - CheckDNS(service *types.ServiceConfig) - CheckDNSOpts(service *types.ServiceConfig) - CheckDNSSearch(service *types.ServiceConfig) - CheckDomainName(service *types.ServiceConfig) - CheckExtraHosts(service *types.ServiceConfig) - CheckHostname(service *types.ServiceConfig) - CheckIpc(service *types.ServiceConfig) - CheckLabels(service *types.ServiceConfig) - CheckLinks(service *types.ServiceConfig) - CheckLogging(service *types.ServiceConfig) - CheckMacAddress(service *types.ServiceConfig) - CheckNetworkMode(service *types.ServiceConfig) - CheckPid(service *types.ServiceConfig) - CheckSysctls(service *types.ServiceConfig) - CheckTmpfs(service *types.ServiceConfig) - CheckUserNSMode(service *types.ServiceConfig) - Errors() []error -} - -// Check the compose model do not use unsupported features and inject sane defaults for ECS deployment -func Check(project *compose.Project) []error { - c := FargateCompatibilityChecker{} - for i, service := range project.Services { - c.CheckService(&service) - project.Services[i] = service - } - return c.errors -} diff --git a/ecs/pkg/amazon/compatibility/check_test.go b/ecs/pkg/amazon/compatibility/check_test.go deleted file mode 100644 index f6589853..00000000 --- a/ecs/pkg/amazon/compatibility/check_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package compatibility - -import ( - "testing" - - "github.com/docker/ecs-plugin/pkg/compose" - "gotest.tools/v3/assert" -) - -func load(t *testing.T, paths ...string) *compose.Project { - options := compose.ProjectOptions{ - Name: t.Name(), - ConfigPaths: paths, - } - project, err := compose.ProjectFromOptions(&options) - assert.NilError(t, err) - return project -} -func TestInvalidNetworkMode(t *testing.T) { - project := load(t, "../backend/testdata/invalid_network_mode.yaml") - err := Check(project) - assert.Error(t, err[0], "'network_mode' \"bridge\" is not supported") -} diff --git a/ecs/pkg/amazon/compatibility/compatibility.go b/ecs/pkg/amazon/compatibility/compatibility.go deleted file mode 100644 index c7d67c26..00000000 --- a/ecs/pkg/amazon/compatibility/compatibility.go +++ /dev/null @@ -1,171 +0,0 @@ -package compatibility - -import ( - "fmt" - - "github.com/aws/aws-sdk-go/service/ecs" - "github.com/compose-spec/compose-go/types" -) - -type FargateCompatibilityChecker struct { - errors []error -} - -func (c *FargateCompatibilityChecker) error(message string, args ...interface{}) { - c.errors = append(c.errors, fmt.Errorf(message, args...)) -} - -func (c *FargateCompatibilityChecker) Errors() []error { - return c.errors -} - -func (c *FargateCompatibilityChecker) CheckService(service *types.ServiceConfig) { - c.CheckCapAdd(service) - c.CheckDNS(service) - c.CheckDNSOpts(service) - c.CheckDNSSearch(service) - c.CheckDomainName(service) - c.CheckExtraHosts(service) - c.CheckHostname(service) - c.CheckIpc(service) - c.CheckLabels(service) - c.CheckLinks(service) - c.CheckLogging(service) - c.CheckMacAddress(service) - c.CheckNetworkMode(service) - c.CheckPid(service) - c.CheckSysctls(service) - c.CheckTmpfs(service) - c.CheckUserNSMode(service) -} - -func (c *FargateCompatibilityChecker) CheckNetworkMode(service *types.ServiceConfig) { - if service.NetworkMode != "" && service.NetworkMode != ecs.NetworkModeAwsvpc { - c.error("'network_mode' %q is not supported", service.NetworkMode) - } - service.NetworkMode = ecs.NetworkModeAwsvpc -} - -func (c *FargateCompatibilityChecker) CheckLinks(service *types.ServiceConfig) { - if len(service.Links) != 0 { - c.error("'links' is not supported") - service.Links = nil - } -} - -func (c *FargateCompatibilityChecker) CheckLogging(service *types.ServiceConfig) { - c.CheckLoggingDriver(service) -} - -func (c *FargateCompatibilityChecker) CheckLoggingDriver(service *types.ServiceConfig) { - if service.LogDriver != "" && service.LogDriver != ecs.LogDriverAwslogs { - c.error("'log_driver' %q is not supported", service.LogDriver) - service.LogDriver = ecs.LogDriverAwslogs - } -} - -func (c *FargateCompatibilityChecker) CheckPid(service *types.ServiceConfig) { - if service.Pid != "" { - c.error("'pid' is not supported") - service.Pid = "" - } -} - -func (c *FargateCompatibilityChecker) CheckUserNSMode(service *types.ServiceConfig) { - if service.UserNSMode != "" { - c.error("'userns_mode' is not supported") - service.UserNSMode = "" - } -} - -func (c *FargateCompatibilityChecker) CheckIpc(service *types.ServiceConfig) { - if service.Ipc != "" { - c.error("'ipc' is not supported") - service.Ipc = "" - } -} - -func (c *FargateCompatibilityChecker) CheckMacAddress(service *types.ServiceConfig) { - if service.MacAddress != "" { - c.error("'mac_address' is not supported") - service.MacAddress = "" - } -} - -func (c *FargateCompatibilityChecker) CheckHostname(service *types.ServiceConfig) { - if service.Hostname != "" { - c.error("'hostname' is not supported") - service.Hostname = "" - } -} - -func (c *FargateCompatibilityChecker) CheckDomainName(service *types.ServiceConfig) { - if service.DomainName != "" { - c.error("'domainname' is not supported") - service.DomainName = "" - } -} - -func (c *FargateCompatibilityChecker) CheckDNSSearch(service *types.ServiceConfig) { - if len(service.DNSSearch) > 0 { - c.error("'dns_search' is not supported") - service.DNSSearch = nil - } -} - -func (c *FargateCompatibilityChecker) CheckDNS(service *types.ServiceConfig) { - if len(service.DNS) > 0 { - c.error("'dns' is not supported") - service.DNS = nil - } -} - -func (c *FargateCompatibilityChecker) CheckDNSOpts(service *types.ServiceConfig) { - if len(service.DNSOpts) > 0 { - c.error("'dns_opt' is not supported") - service.DNSOpts = nil - } -} - -func (c *FargateCompatibilityChecker) CheckExtraHosts(service *types.ServiceConfig) { - if len(service.ExtraHosts) > 0 { - c.error("'extra_hosts' is not supported") - service.ExtraHosts = nil - } -} - -func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) { - for i, v := range service.CapAdd { - if v != "SYS_PTRACE" { - c.error("'cap_add' %s is not supported", v) - l := len(service.CapAdd) - service.CapAdd[i] = service.CapAdd[l-1] - service.CapAdd = service.CapAdd[:l-1] - } - } -} - -func (c *FargateCompatibilityChecker) CheckTmpfs(service *types.ServiceConfig) { - if len(service.Tmpfs) > 0 { - c.error("'tmpfs' is not supported") - service.Tmpfs = nil - } -} - -func (c *FargateCompatibilityChecker) CheckSysctls(service *types.ServiceConfig) { - if len(service.Sysctls) > 0 { - c.error("'sysctls' is not supported") - service.Sysctls = nil - } -} - -func (c *FargateCompatibilityChecker) CheckLabels(service *types.ServiceConfig) { - for k, v := range service.Labels { - if v == "" { - c.error("'labels' with an empty value is not supported") - delete(service.Labels, k) - } - } -} - -var _ Checker = &FargateCompatibilityChecker{} diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index 137a4339..1e3e61ad 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -5,7 +5,7 @@ import ( cf "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/awslabs/goformation/v4/cloudformation" - "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/docker/ecs-plugin/pkg/compose" ) //go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon" -package=amazon . API @@ -38,19 +38,19 @@ type downAPI interface { } type logsAPI interface { - GetLogs(ctx context.Context, name string, consumer types.LogConsumer) error + GetLogs(ctx context.Context, name string, consumer compose.LogConsumer) error } type secretsAPI interface { - CreateSecret(ctx context.Context, secret types.Secret) (string, error) - InspectSecret(ctx context.Context, id string) (types.Secret, error) - ListSecrets(ctx context.Context) ([]types.Secret, error) + CreateSecret(ctx context.Context, secret compose.Secret) (string, error) + InspectSecret(ctx context.Context, id string) (compose.Secret, error) + ListSecrets(ctx context.Context) ([]compose.Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error } type listAPI interface { ListTasks(ctx context.Context, cluster string, name string) ([]string, error) - DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]types.TaskStatus, error) + DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]compose.TaskStatus, error) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error) } diff --git a/ecs/pkg/amazon/sdk/api_mock.go b/ecs/pkg/amazon/sdk/api_mock.go index 07ce79e7..0fe42fb2 100644 --- a/ecs/pkg/amazon/sdk/api_mock.go +++ b/ecs/pkg/amazon/sdk/api_mock.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" + "github.com/docker/ecs-plugin/pkg/compose" + cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" - btypes "github.com/docker/ecs-plugin/pkg/amazon/types" gomock "github.com/golang/mock/gomock" ) @@ -53,7 +54,7 @@ func (mr *MockAPIMockRecorder) ClusterExists(arg0, arg1 interface{}) *gomock.Cal } // CreateSecret mocks base method -func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 btypes.Secret) (string, error) { +func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 compose.Secret) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateSecret", arg0, arg1) ret0, _ := ret[0].(string) @@ -139,14 +140,14 @@ func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomo } // DescribeTasks mocks base method -func (m *MockAPI) DescribeTasks(arg0 context.Context, arg1 string, arg2 ...string) ([]btypes.TaskStatus, error) { +func (m *MockAPI) DescribeTasks(arg0 context.Context, arg1 string, arg2 ...string) ([]compose.TaskStatus, error) { m.ctrl.T.Helper() varargs := []interface{}{arg0, arg1} for _, a := range arg2 { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "DescribeTasks", varargs...) - ret0, _ := ret[0].([]btypes.TaskStatus) + ret0, _ := ret[0].([]compose.TaskStatus) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -174,7 +175,7 @@ func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call { } // GetLogs mocks base method -func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 btypes.LogConsumer) error { +func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 compose.LogConsumer) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetLogs", arg0, arg1, arg2) ret0, _ := ret[0].(error) @@ -238,10 +239,10 @@ func (mr *MockAPIMockRecorder) GetSubNets(arg0, arg1 interface{}) *gomock.Call { } // InspectSecret mocks base method -func (m *MockAPI) InspectSecret(arg0 context.Context, arg1 string) (btypes.Secret, error) { +func (m *MockAPI) InspectSecret(arg0 context.Context, arg1 string) (compose.Secret, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InspectSecret", arg0, arg1) - ret0, _ := ret[0].(btypes.Secret) + ret0, _ := ret[0].(compose.Secret) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -253,10 +254,10 @@ func (mr *MockAPIMockRecorder) InspectSecret(arg0, arg1 interface{}) *gomock.Cal } // ListSecrets mocks base method -func (m *MockAPI) ListSecrets(arg0 context.Context) ([]btypes.Secret, error) { +func (m *MockAPI) ListSecrets(arg0 context.Context) ([]compose.Secret, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListSecrets", arg0) - ret0, _ := ret[0].([]btypes.Secret) + ret0, _ := ret[0].([]compose.Secret) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/ecs/pkg/amazon/sdk/convert.go b/ecs/pkg/amazon/sdk/convert.go index 8874aa83..3e2b861f 100644 --- a/ecs/pkg/amazon/sdk/convert.go +++ b/ecs/pkg/amazon/sdk/convert.go @@ -13,11 +13,10 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/compose-spec/compose-go/types" "github.com/docker/cli/opts" - t "github.com/docker/ecs-plugin/pkg/amazon/types" "github.com/docker/ecs-plugin/pkg/compose" ) -func Convert(project *compose.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { +func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { cpu, mem, err := toLimits(service) if err != nil { return nil, err @@ -318,8 +317,8 @@ func getImage(image string) string { func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name - for key, value := range service.Extras { - if key == t.ExtensionPullCredentials { + for key, value := range service.Extensions { + if key == compose.ExtensionPullCredentials { return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: value.(string)} } } diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 1e1cb849..3f0ed929 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -23,10 +23,8 @@ import ( "github.com/aws/aws-sdk-go/service/secretsmanager" "github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface" cf "github.com/awslabs/goformation/v4/cloudformation" + "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" - - "github.com/docker/ecs-plugin/pkg/amazon/types" - t "github.com/docker/ecs-plugin/pkg/amazon/types" ) type sdk struct { @@ -189,9 +187,9 @@ func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) StackName: aws.String(name), } switch operation { - case t.StackCreate: + case compose.StackCreate: return s.CF.WaitUntilStackCreateCompleteWithContext(ctx, input) - case t.StackDelete: + case compose.StackDelete: return s.CF.WaitUntilStackDeleteCompleteWithContext(ctx, input) default: return fmt.Errorf("internal error: unexpected stack operation %d", operation) @@ -236,7 +234,7 @@ func (s sdk) DeleteStack(ctx context.Context, name string) error { return err } -func (s sdk) CreateSecret(ctx context.Context, secret t.Secret) (string, error) { +func (s sdk) CreateSecret(ctx context.Context, secret compose.Secret) (string, error) { logrus.Debug("Create secret " + secret.Name) secretStr, err := secret.GetCredString() if err != nil { @@ -254,17 +252,17 @@ func (s sdk) CreateSecret(ctx context.Context, secret t.Secret) (string, error) return *response.ARN, nil } -func (s sdk) InspectSecret(ctx context.Context, id string) (t.Secret, error) { +func (s sdk) InspectSecret(ctx context.Context, id string) (compose.Secret, error) { logrus.Debug("Inspect secret " + id) response, err := s.SM.DescribeSecret(&secretsmanager.DescribeSecretInput{SecretId: &id}) if err != nil { - return t.Secret{}, err + return compose.Secret{}, err } labels := map[string]string{} for _, tag := range response.Tags { labels[*tag.Key] = *tag.Value } - secret := t.Secret{ + secret := compose.Secret{ ID: *response.ARN, Name: *response.Name, Labels: labels, @@ -275,14 +273,14 @@ func (s sdk) InspectSecret(ctx context.Context, id string) (t.Secret, error) { return secret, nil } -func (s sdk) ListSecrets(ctx context.Context) ([]t.Secret, error) { +func (s sdk) ListSecrets(ctx context.Context) ([]compose.Secret, error) { logrus.Debug("List secrets ...") response, err := s.SM.ListSecrets(&secretsmanager.ListSecretsInput{}) if err != nil { - return []t.Secret{}, err + return []compose.Secret{}, err } - var secrets []t.Secret + var secrets []compose.Secret for _, sec := range response.SecretList { @@ -294,7 +292,7 @@ func (s sdk) ListSecrets(ctx context.Context) ([]t.Secret, error) { if sec.Description != nil { description = *sec.Description } - secrets = append(secrets, t.Secret{ + secrets = append(secrets, compose.Secret{ ID: *sec.ARN, Name: *sec.Name, Labels: labels, @@ -311,7 +309,7 @@ func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { return err } -func (s sdk) GetLogs(ctx context.Context, name string, consumer types.LogConsumer) error { +func (s sdk) GetLogs(ctx context.Context, name string, consumer compose.LogConsumer) error { logGroup := fmt.Sprintf("/docker-compose/%s", name) var startTime int64 for { @@ -357,7 +355,7 @@ func (s sdk) ListTasks(ctx context.Context, cluster string, service string) ([]s return arns, nil } -func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]t.TaskStatus, error) { +func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]compose.TaskStatus, error) { tasks, err := s.ECS.DescribeTasksWithContext(ctx, &ecs.DescribeTasksInput{ Cluster: aws.String(cluster), Tasks: aws.StringSlice(arns), @@ -365,7 +363,7 @@ func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) if err != nil { return nil, err } - result := []t.TaskStatus{} + result := []compose.TaskStatus{} for _, task := range tasks.Tasks { var networkInterface string for _, attachement := range task.Attachments { @@ -377,7 +375,7 @@ func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) } } } - result = append(result, t.TaskStatus{ + result = append(result, compose.TaskStatus{ State: *task.LastStatus, Service: strings.Replace(*task.Group, "service:", "", 1), NetworkInterface: networkInterface, diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 6d84ccec..0d4877a8 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -4,19 +4,20 @@ import ( "context" "github.com/awslabs/goformation/v4/cloudformation" - "github.com/docker/ecs-plugin/pkg/amazon/types" + "github.com/compose-spec/compose-go/cli" + "github.com/compose-spec/compose-go/types" ) type API interface { - Up(ctx context.Context, options ProjectOptions) error - Down(ctx context.Context, options ProjectOptions) error + Up(ctx context.Context, options cli.ProjectOptions) error + Down(ctx context.Context, options cli.ProjectOptions) error - Convert(project *Project) (*cloudformation.Template, error) + Convert(project *types.Project) (*cloudformation.Template, error) Logs(ctx context.Context, projectName string) error - Ps(background context.Context, project *Project) ([]types.TaskStatus, error) + Ps(background context.Context, project *types.Project) ([]TaskStatus, error) - CreateSecret(ctx context.Context, secret types.Secret) (string, error) - InspectSecret(ctx context.Context, id string) (types.Secret, error) - ListSecrets(ctx context.Context) ([]types.Secret, error) + CreateSecret(ctx context.Context, secret Secret) (string, error) + InspectSecret(ctx context.Context, id string) (Secret, error) + ListSecrets(ctx context.Context) ([]Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error } diff --git a/ecs/pkg/compose/normalize.go b/ecs/pkg/compose/normalize.go deleted file mode 100644 index 3e4809d1..00000000 --- a/ecs/pkg/compose/normalize.go +++ /dev/null @@ -1,89 +0,0 @@ -package compose - -import ( - "fmt" - - "github.com/compose-spec/compose-go/types" - "github.com/sirupsen/logrus" -) - -// Normalize a compose-go model to move deprecated attributes to canonical position, and introduce implicit defaults -// FIXME move this to compose-go -func Normalize(model *types.Config) error { - if len(model.Networks) == 0 { - // Compose application model implies a default network if none is explicitly set. - model.Networks["default"] = types.NetworkConfig{ - Name: "default", - } - } - - for i, s := range model.Services { - if len(s.Networks) == 0 { - // Service without explicit network attachment are implicitly exposed on default network - s.Networks = map[string]*types.ServiceNetworkConfig{"default": nil} - } - - for i, p := range s.Ports { - if p.Published == 0 { - p.Published = p.Target - s.Ports[i] = p - } - } - - if s.LogDriver != "" { - logrus.Warn("`log_driver` is deprecated. Use the `logging` attribute") - if s.Logging == nil { - s.Logging = &types.LoggingConfig{} - } - if s.Logging.Driver == "" { - s.Logging.Driver = s.LogDriver - } else { - return fmt.Errorf("can't use both 'log_driver' (deprecated) and 'logging.driver'") - } - } - if len(s.LogOpt) != 0 { - logrus.Warn("`log_opts` is deprecated. Use the `logging` attribute") - if s.Logging == nil { - s.Logging = &types.LoggingConfig{} - } - for k, v := range s.LogOpt { - if _, ok := s.Logging.Options[k]; !ok { - s.Logging.Options[k] = v - } else { - return fmt.Errorf("can't use both 'log_opt' (deprecated) and 'logging.options'") - } - } - } - model.Services[i] = s - } - - for i, n := range model.Networks { - if n.Name == "" { - n.Name = i - model.Networks[i] = n - } - } - - for i, v := range model.Volumes { - if v.Name == "" { - v.Name = i - model.Volumes[i] = v - } - } - - for i, c := range model.Configs { - if c.Name == "" { - c.Name = i - model.Configs[i] = c - } - } - - for i, s := range model.Secrets { - if s.Name == "" { - s.Name = i - model.Secrets[i] = s - } - } - - return nil -} diff --git a/ecs/pkg/compose/project.go b/ecs/pkg/compose/project.go deleted file mode 100644 index a90bba07..00000000 --- a/ecs/pkg/compose/project.go +++ /dev/null @@ -1,170 +0,0 @@ -package compose - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/compose-spec/compose-go/loader" - "github.com/compose-spec/compose-go/types" - "github.com/sirupsen/logrus" -) - -type Project struct { - types.Config - projectDir string - Name string `yaml:"-" json:"-"` -} - -func NewProject(config types.ConfigDetails, name string) (*Project, error) { - model, err := loader.Load(config) - if err != nil { - return nil, err - } - - err = Normalize(model) - if err != nil { - return nil, err - } - - p := Project{ - Config: *model, - projectDir: config.WorkingDir, - Name: name, - } - return &p, nil -} - -// projectFromOptions load a compose project based on command line options -func ProjectFromOptions(options *ProjectOptions) (*Project, error) { - configPath, err := getConfigPathFromOptions(options) - if err != nil { - return nil, err - } - - name := options.Name - if name == "" { - name = os.Getenv("COMPOSE_PROJECT_NAME") - } - - workingDir := filepath.Dir(configPath[0]) - - if name == "" { - r := regexp.MustCompile(`[^a-z0-9\\-_]+`) - name = r.ReplaceAllString(strings.ToLower(filepath.Base(workingDir)), "") - } - - configs, err := parseConfigs(configPath) - if err != nil { - return nil, err - } - - return NewProject(types.ConfigDetails{ - WorkingDir: workingDir, - ConfigFiles: configs, - Environment: environment(), - }, name) -} - -func getConfigPathFromOptions(options *ProjectOptions) ([]string, error) { - paths := []string{} - pwd, err := os.Getwd() - if err != nil { - return nil, err - } - - if len(options.ConfigPaths) != 0 { - for _, f := range options.ConfigPaths { - if f == "-" { - paths = append(paths, f) - continue - } - if !filepath.IsAbs(f) { - f = filepath.Join(pwd, f) - } - if _, err := os.Stat(f); err != nil { - return nil, err - } - paths = append(paths, f) - } - return paths, nil - } - - sep := os.Getenv("COMPOSE_FILE_SEPARATOR") - if sep == "" { - sep = string(os.PathListSeparator) - } - f := os.Getenv("COMPOSE_FILE") - if f != "" { - return strings.Split(f, sep), nil - } - - for { - candidates := []string{} - for _, n := range SupportedFilenames { - f := filepath.Join(pwd, n) - if _, err := os.Stat(f); err == nil { - candidates = append(candidates, f) - } - } - if len(candidates) > 0 { - winner := candidates[0] - if len(candidates) > 1 { - logrus.Warnf("Found multiple config files with supported names: %s", strings.Join(candidates, ", ")) - logrus.Warnf("Using %s\n", winner) - } - return []string{winner}, nil - } - parent := filepath.Dir(pwd) - if parent == pwd { - return nil, fmt.Errorf("Can't find a suitable configuration file in this directory or any parent. Are you in the right directory?") - } - pwd = parent - } -} - -var SupportedFilenames = []string{"compose.yaml", "compose.yml", "docker-compose.yml", "docker-compose.yaml"} - -func parseConfigs(configPaths []string) ([]types.ConfigFile, error) { - files := []types.ConfigFile{} - for _, f := range configPaths { - var ( - b []byte - err error - ) - if f == "-" { - b, err = ioutil.ReadAll(os.Stdin) - } else { - if _, err := os.Stat(f); err != nil { - return nil, err - } - b, err = ioutil.ReadFile(f) - } - if err != nil { - return nil, err - } - config, err := loader.ParseYAML(b) - if err != nil { - return nil, err - } - files = append(files, types.ConfigFile{Filename: f, Config: config}) - } - return files, nil -} - -func environment() map[string]string { - return getAsEqualsMap(os.Environ()) -} - -// getAsEqualsMap split key=value formatted strings into a key : value map -func getAsEqualsMap(em []string) map[string]string { - m := make(map[string]string) - for _, v := range em { - kv := strings.SplitN(v, "=", 2) - m[kv[0]] = kv[1] - } - return m -} diff --git a/ecs/pkg/compose/project_test.go b/ecs/pkg/compose/project_test.go deleted file mode 100644 index 733f34f0..00000000 --- a/ecs/pkg/compose/project_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package compose - -import ( - "os" - "testing" - - "gotest.tools/v3/assert" -) - -func Test_project_name(t *testing.T) { - p, err := ProjectFromOptions(&ProjectOptions{ - Name: "my_project", - ConfigPaths: []string{"testdata/simple/compose.yaml"}, - }) - assert.NilError(t, err) - assert.Equal(t, p.Name, "my_project") - - p, err = ProjectFromOptions(&ProjectOptions{ - Name: "", - ConfigPaths: []string{"testdata/simple/compose.yaml"}, - }) - assert.NilError(t, err) - assert.Equal(t, p.Name, "simple") - - os.Setenv("COMPOSE_PROJECT_NAME", "my_project_from_env") - p, err = ProjectFromOptions(&ProjectOptions{ - Name: "", - ConfigPaths: []string{"testdata/simple/compose.yaml"}, - }) - assert.NilError(t, err) - assert.Equal(t, p.Name, "my_project_from_env") -} - -func Test_project_from_set_of_files(t *testing.T) { - p, err := ProjectFromOptions(&ProjectOptions{ - Name: "my_project", - ConfigPaths: []string{ - "testdata/simple/compose.yaml", - "testdata/simple/compose-with-overrides.yaml", - }, - }) - assert.NilError(t, err) - service, err := p.GetService("simple") - assert.NilError(t, err) - assert.Equal(t, service.Image, "haproxy") -} diff --git a/ecs/pkg/amazon/types/types.go b/ecs/pkg/compose/types.go similarity index 98% rename from ecs/pkg/amazon/types/types.go rename to ecs/pkg/compose/types.go index f6815955..13326365 100644 --- a/ecs/pkg/amazon/types/types.go +++ b/ecs/pkg/compose/types.go @@ -1,4 +1,4 @@ -package types +package compose import "encoding/json" diff --git a/ecs/pkg/amazon/types/x.go b/ecs/pkg/compose/x.go similarity index 92% rename from ecs/pkg/amazon/types/x.go rename to ecs/pkg/compose/x.go index d8b8e011..7a2a5bd0 100644 --- a/ecs/pkg/amazon/types/x.go +++ b/ecs/pkg/compose/x.go @@ -1,4 +1,4 @@ -package types +package compose const ( ExtensionSecurityGroup = "x-aws-securitygroup" From 874be0873d425a3ab7c3ec687835338767e57d0c Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 15 Jun 2020 15:52:47 +0200 Subject: [PATCH 129/205] generate code inside Docker container Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 2 ++ ecs/builder.Makefile | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ecs/Dockerfile b/ecs/Dockerfile index 8d18ddfe..dead5829 100644 --- a/ecs/Dockerfile +++ b/ecs/Dockerfile @@ -18,6 +18,8 @@ COPY . . FROM base AS make-plugin ARG TARGETOS ARG TARGETARCH +RUN apk add build-base +RUN GO111MODULE=on go get github.com/golang/mock/mockgen@latest RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ GOOS=${TARGETOS} \ diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile index 8b6920ba..18bd2da2 100644 --- a/ecs/builder.Makefile +++ b/ecs/builder.Makefile @@ -20,7 +20,10 @@ all: build clean: rm -rf dist/ -build: +generate: pkg/amazon/sdk/api_mock.go + go generate ./... + +build: generate $(GO_BUILD) -v -o $(BINARY_WITH_EXTENSION) cmd/main/main.go cross: From ed262a0461d6e9b897b0f75b6336ee7ba89dcf4a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 23 Jun 2020 10:27:27 +0200 Subject: [PATCH 130/205] Generate mock inside a container Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/sdk/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index 1e3e61ad..feaa1eb7 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -8,7 +8,7 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) -//go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon" -package=amazon . API +//go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon/sdk" -package=sdk . API type API interface { downAPI From 5e1f40b752937fb1c9b9e5787eca56fcc35c0600 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 29 Jun 2020 09:12:43 +0200 Subject: [PATCH 131/205] Document required AWS permissions Signed-off-by: Nicolas De Loof --- ecs/docs/requirements.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 ecs/docs/requirements.md diff --git a/ecs/docs/requirements.md b/ecs/docs/requirements.md new file mode 100644 index 00000000..d142de4f --- /dev/null +++ b/ecs/docs/requirements.md @@ -0,0 +1,31 @@ +## Requirements + +This plugin relies on AWS API credentials, using the same configuration files as +the AWS command line. + +Such credentials can be configured by the `docker ecs setup` command, either by +selecting an existing AWS CLI profile from existing config files, or by creating +one passing an AWS access key ID and secret access key. + +## Permissions + +AWS accounts (or IAM roles) used with the ECS plugin require following permissions: + +- ec2:DescribeSubnets +- ec2:DescribeVpcs +- iam:CreateServiceLinkedRole +- iam:AttachRolePolicy +- cloudformation:* +- ecs:* +- logs:* +- servicediscovery:* +- elasticloadbalancing:* + + +## Okta support + +For those relying on [aws-okta](https://github.com/segmentio/aws-okta) to access a managed AWS account +(as we do at Docker), you can populate your aws config files with temporary access tokens using: +```shell script +aws-okta write-to-credentials ~/.aws/credentials +``` From cb74f7924e5bf89e67f55229c04fe67b2137643e Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 15 Jun 2020 17:48:56 +0200 Subject: [PATCH 132/205] Don't define service resource name if we do, CloudFormation can't update resource and changeset fail with "CloudFormation cannot update a stack when a custom-named resource requires replacing" Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 1 - ecs/pkg/amazon/sdk/sdk.go | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 5cdbcdc2..c27f6531 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -193,7 +193,6 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro }, }, SchedulingStrategy: ecsapi.SchedulingStrategyReplica, - ServiceName: service.Name, ServiceRegistries: []ecs.Service_ServiceRegistry{serviceRegistry}, Tags: []tags.Tag{ { diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 3f0ed929..4d882869 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -182,6 +182,7 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template }) return err } + func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) error { input := &cloudformation.DescribeStacksInput{ StackName: aws.String(name), From a1eba59a467578b807b23d7a274252f86203dc65 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 17 Jun 2020 10:20:12 +0200 Subject: [PATCH 133/205] `ps` do list services, not containers Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 10 +- ecs/pkg/amazon/backend/cloudformation.go | 3 +- ecs/pkg/amazon/{sdk => backend}/convert.go | 2 +- ecs/pkg/amazon/backend/list.go | 52 ++------ .../simple-cloudformation-conversion.golden | 1 - ...formation-with-overrides-conversion.golden | 1 - ecs/pkg/amazon/sdk/api.go | 4 +- ecs/pkg/amazon/sdk/api_mock.go | 118 ++++++------------ ecs/pkg/amazon/sdk/sdk.go | 51 +++----- ecs/pkg/compose/api.go | 2 +- ecs/pkg/compose/types.go | 13 +- ecs/tests/e2e_deploy_services_test.go | 11 +- 12 files changed, 91 insertions(+), 177 deletions(-) rename ecs/pkg/amazon/{sdk => backend}/convert.go (99%) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index b3489bae..ed488a89 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -101,15 +101,15 @@ func PsCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Co if err != nil { return err } - tasks, err := backend.Ps(context.Background(), project) + status, err := backend.Ps(context.Background(), project) if err != nil { return err } - printSection(os.Stdout, len(tasks), func(w io.Writer) { - for _, task := range tasks { - fmt.Fprintf(w, "%s\t%s\t%s\n", task.Name, task.State, strings.Join(task.Ports, " ")) + printSection(os.Stdout, len(status), func(w io.Writer) { + for _, service := range status { + fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\n", service.ID, service.Name, service.Replicas, service.Desired, strings.Join(service.Ports, " ")) } - }, "NAME", "STATE", "PORTS") + }, "ID", "NAME", "REPLICAS", "PORTS") return nil }), } diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index c27f6531..f4111240 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -18,7 +18,6 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/compose-spec/compose-go/compatibility" "github.com/compose-spec/compose-go/types" - sdk "github.com/docker/ecs-plugin/pkg/amazon/sdk" "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" ) @@ -120,7 +119,7 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro for _, service := range project.Services { - definition, err := sdk.Convert(project, service) + definition, err := Convert(project, service) if err != nil { return nil, err } diff --git a/ecs/pkg/amazon/sdk/convert.go b/ecs/pkg/amazon/backend/convert.go similarity index 99% rename from ecs/pkg/amazon/sdk/convert.go rename to ecs/pkg/amazon/backend/convert.go index 3e2b861f..0eace3a6 100644 --- a/ecs/pkg/amazon/sdk/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -1,4 +1,4 @@ -package sdk +package backend import ( "fmt" diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index 4385eae5..c13d2f90 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -3,61 +3,29 @@ package backend import ( "context" "fmt" - "sort" - "strings" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) Ps(ctx context.Context, project *types.Project) ([]compose.TaskStatus, error) { +func (b *Backend) Ps(ctx context.Context, project *types.Project) ([]compose.ServiceStatus, error) { cluster := b.Cluster if cluster == "" { cluster = project.Name } - arns := []string{} + + status := []compose.ServiceStatus{} for _, service := range project.Services { - tasks, err := b.api.ListTasks(ctx, cluster, service.Name) + desc, err := b.api.DescribeService(ctx, cluster, service.Name) if err != nil { - return []compose.TaskStatus{}, err + return nil, err } - arns = append(arns, tasks...) - } - if len(arns) == 0 { - return []compose.TaskStatus{}, nil - } - - tasks, err := b.api.DescribeTasks(ctx, cluster, arns...) - if err != nil { - return []compose.TaskStatus{}, err - } - - networkInterfaces := []string{} - for _, t := range tasks { - if t.NetworkInterface != "" { - networkInterfaces = append(networkInterfaces, t.NetworkInterface) - } - } - publicIps, err := b.api.GetPublicIPs(ctx, networkInterfaces...) - if err != nil { - return []compose.TaskStatus{}, err - } - - sort.Slice(tasks, func(i, j int) bool { - return strings.Compare(tasks[i].Service, tasks[j].Service) < 0 - }) - - for i, task := range tasks { ports := []string{} - s, err := project.GetService(task.Service) - if err != nil { - return []compose.TaskStatus{}, err + for _, p := range service.Ports { + ports = append(ports, fmt.Sprintf("*:%d->%d/%s", p.Published, p.Target, p.Protocol)) } - for _, p := range s.Ports { - ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", publicIps[task.NetworkInterface], p.Published, p.Target, p.Protocol)) - } - tasks[i].Name = s.Name - tasks[i].Ports = ports + desc.Ports = ports + status = append(status, desc) } - return tasks, nil + return status, nil } diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden index 0003b38e..b9fd6a04 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden @@ -104,7 +104,6 @@ } }, "SchedulingStrategy": "REPLICA", - "ServiceName": "simple", "ServiceRegistries": [ { "RegistryArn": { diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 07c47765..0f93cbd5 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -104,7 +104,6 @@ } }, "SchedulingStrategy": "REPLICA", - "ServiceName": "simple", "ServiceRegistries": [ { "RegistryArn": { diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index feaa1eb7..1ad68afa 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -49,9 +49,7 @@ type secretsAPI interface { } type listAPI interface { - ListTasks(ctx context.Context, cluster string, name string) ([]string, error) - DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]compose.TaskStatus, error) - GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error) + DescribeService(ctx context.Context, cluster string, name string) (compose.ServiceStatus, error) } type waitAPI interface { diff --git a/ecs/pkg/amazon/sdk/api_mock.go b/ecs/pkg/amazon/sdk/api_mock.go index 0fe42fb2..dda73b71 100644 --- a/ecs/pkg/amazon/sdk/api_mock.go +++ b/ecs/pkg/amazon/sdk/api_mock.go @@ -1,18 +1,16 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/docker/ecs-plugin/pkg/amazon (interfaces: API) +// Source: github.com/docker/ecs-plugin/pkg/amazon/sdk (interfaces: API) -// Package amazon is a generated GoMock package. +// Package sdk is a generated GoMock package. package sdk import ( context "context" - reflect "reflect" - - "github.com/docker/ecs-plugin/pkg/compose" - cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" + compose "github.com/docker/ecs-plugin/pkg/compose" gomock "github.com/golang/mock/gomock" + reflect "reflect" ) // MockAPI is a mock of API interface @@ -124,6 +122,21 @@ func (mr *MockAPIMockRecorder) DeleteStack(arg0, arg1 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), arg0, arg1) } +// DescribeService mocks base method +func (m *MockAPI) DescribeService(arg0 context.Context, arg1, arg2 string) (compose.ServiceStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeService", arg0, arg1, arg2) + ret0, _ := ret[0].(compose.ServiceStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeService indicates an expected call of DescribeService +func (mr *MockAPIMockRecorder) DescribeService(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeService", reflect.TypeOf((*MockAPI)(nil).DescribeService), arg0, arg1, arg2) +} + // DescribeStackEvents mocks base method func (m *MockAPI) DescribeStackEvents(arg0 context.Context, arg1 string) ([]*cloudformation.StackEvent, error) { m.ctrl.T.Helper() @@ -139,26 +152,6 @@ func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0, arg1) } -// DescribeTasks mocks base method -func (m *MockAPI) DescribeTasks(arg0 context.Context, arg1 string, arg2 ...string) ([]compose.TaskStatus, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "DescribeTasks", varargs...) - ret0, _ := ret[0].([]compose.TaskStatus) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DescribeTasks indicates an expected call of DescribeTasks -func (mr *MockAPIMockRecorder) DescribeTasks(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTasks", reflect.TypeOf((*MockAPI)(nil).DescribeTasks), varargs...) -} - // GetDefaultVPC mocks base method func (m *MockAPI) GetDefaultVPC(arg0 context.Context) (string, error) { m.ctrl.T.Helper() @@ -174,6 +167,21 @@ func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC), arg0) } +// GetLoadBalancerARN mocks base method +func (m *MockAPI) GetLoadBalancerARN(arg0 context.Context, arg1 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoadBalancerARN", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLoadBalancerARN indicates an expected call of GetLoadBalancerARN +func (mr *MockAPIMockRecorder) GetLoadBalancerARN(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoadBalancerARN", reflect.TypeOf((*MockAPI)(nil).GetLoadBalancerARN), arg0, arg1) +} + // GetLogs mocks base method func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 compose.LogConsumer) error { m.ctrl.T.Helper() @@ -188,26 +196,6 @@ func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1, arg2 interface{}) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1, arg2) } -// GetPublicIPs mocks base method -func (m *MockAPI) GetPublicIPs(arg0 context.Context, arg1 ...string) (map[string]string, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GetPublicIPs", varargs...) - ret0, _ := ret[0].(map[string]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetPublicIPs indicates an expected call of GetPublicIPs -func (mr *MockAPIMockRecorder) GetPublicIPs(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPublicIPs", reflect.TypeOf((*MockAPI)(nil).GetPublicIPs), varargs...) -} - // GetStackID mocks base method func (m *MockAPI) GetStackID(arg0 context.Context, arg1 string) (string, error) { m.ctrl.T.Helper() @@ -268,19 +256,19 @@ func (mr *MockAPIMockRecorder) ListSecrets(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSecrets", reflect.TypeOf((*MockAPI)(nil).ListSecrets), arg0) } -// ListTasks mocks base method -func (m *MockAPI) ListTasks(arg0 context.Context, arg1, arg2 string) ([]string, error) { +// LoadBalancerExists mocks base method +func (m *MockAPI) LoadBalancerExists(arg0 context.Context, arg1 string) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListTasks", arg0, arg1, arg2) - ret0, _ := ret[0].([]string) + ret := m.ctrl.Call(m, "LoadBalancerExists", arg0, arg1) + ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListTasks indicates an expected call of ListTasks -func (mr *MockAPIMockRecorder) ListTasks(arg0, arg1, arg2 interface{}) *gomock.Call { +// LoadBalancerExists indicates an expected call of LoadBalancerExists +func (mr *MockAPIMockRecorder) LoadBalancerExists(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTasks", reflect.TypeOf((*MockAPI)(nil).ListTasks), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerExists", reflect.TypeOf((*MockAPI)(nil).LoadBalancerExists), arg0, arg1) } // StackExists mocks base method @@ -326,27 +314,3 @@ func (mr *MockAPIMockRecorder) WaitStackComplete(arg0, arg1, arg2 interface{}) * mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitStackComplete", reflect.TypeOf((*MockAPI)(nil).WaitStackComplete), arg0, arg1, arg2) } - -// LoadBalancerExists mocks base method -func (m *MockAPI) LoadBalancerExists(arg0 context.Context, arg1 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadBalancerExists", arg0, arg1) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// LoadBalancerExists indicates an expected call of VpcExists -func (mr *MockAPIMockRecorder) LoadBalancerExists(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerExists", reflect.TypeOf((*MockAPI)(nil).LoadBalancerExists), arg0, arg1) -} - -// GetLoadBalancerARN mocks base method -func (m *MockAPI) GetLoadBalancerARN(arg0 context.Context, arg1 string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLoadBalancerARN", arg0) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 4d882869..3cc2c0fa 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -341,10 +341,26 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer compose.LogConsu } } -func (s sdk) ListTasks(ctx context.Context, cluster string, service string) ([]string, error) { +func (s sdk) DescribeService(ctx context.Context, cluster string, name string) (compose.ServiceStatus, error) { + services, err := s.ECS.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{ + Cluster: aws.String(cluster), + Services: aws.StringSlice([]string{name}), + }) + if err != nil { + return compose.ServiceStatus{}, err + } + return compose.ServiceStatus{ + ID: *services.Services[0].ServiceName, + Name: name, + Replicas: int(*services.Services[0].RunningCount), + Desired: int(*services.Services[0].DesiredCount), + }, nil +} + +func (s sdk) ListTasks(ctx context.Context, cluster string, family string) ([]string, error) { tasks, err := s.ECS.ListTasksWithContext(ctx, &ecs.ListTasksInput{ - Cluster: aws.String(cluster), - ServiceName: aws.String(service), + Cluster: aws.String(cluster), + Family: aws.String(family), }) if err != nil { return nil, err @@ -356,35 +372,6 @@ func (s sdk) ListTasks(ctx context.Context, cluster string, service string) ([]s return arns, nil } -func (s sdk) DescribeTasks(ctx context.Context, cluster string, arns ...string) ([]compose.TaskStatus, error) { - tasks, err := s.ECS.DescribeTasksWithContext(ctx, &ecs.DescribeTasksInput{ - Cluster: aws.String(cluster), - Tasks: aws.StringSlice(arns), - }) - if err != nil { - return nil, err - } - result := []compose.TaskStatus{} - for _, task := range tasks.Tasks { - var networkInterface string - for _, attachement := range task.Attachments { - if *attachement.Type == "ElasticNetworkInterface" { - for _, pair := range attachement.Details { - if *pair.Name == "networkInterfaceId" { - networkInterface = *pair.Value - } - } - } - } - result = append(result, compose.TaskStatus{ - State: *task.LastStatus, - Service: strings.Replace(*task.Group, "service:", "", 1), - NetworkInterface: networkInterface, - }) - } - return result, nil -} - func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error) { desc, err := s.EC2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ NetworkInterfaceIds: aws.StringSlice(interfaces), diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 0d4877a8..64e7e5c8 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -14,7 +14,7 @@ type API interface { Convert(project *types.Project) (*cloudformation.Template, error) Logs(ctx context.Context, projectName string) error - Ps(background context.Context, project *types.Project) ([]TaskStatus, error) + Ps(background context.Context, project *types.Project) ([]ServiceStatus, error) CreateSecret(ctx context.Context, secret Secret) (string, error) InspectSecret(ctx context.Context, id string) (Secret, error) diff --git a/ecs/pkg/compose/types.go b/ecs/pkg/compose/types.go index 13326365..ae56e65e 100644 --- a/ecs/pkg/compose/types.go +++ b/ecs/pkg/compose/types.go @@ -2,13 +2,12 @@ package compose import "encoding/json" -type TaskStatus struct { - Name string - State string - Service string - NetworkInterface string - PublicIP string - Ports []string +type ServiceStatus struct { + ID string + Name string + Replicas int + Desired int + Ports []string } const ( diff --git a/ecs/tests/e2e_deploy_services_test.go b/ecs/tests/e2e_deploy_services_test.go index c8a2242c..e55fc93b 100644 --- a/ecs/tests/e2e_deploy_services_test.go +++ b/ecs/tests/e2e_deploy_services_test.go @@ -6,9 +6,10 @@ import ( "context" "testing" + "github.com/docker/ecs-plugin/pkg/amazon/sdk" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - "github.com/docker/ecs-plugin/pkg/amazon" "github.com/docker/ecs-plugin/pkg/docker" "gotest.tools/v3/assert" "gotest.tools/v3/fs" @@ -47,11 +48,11 @@ func composeUpSimpleService(t *testing.T, cmd icmd.Cmd, awsContext docker.AwsCon }, }) assert.NilError(t, err) - sdk := amazon.NewAPI(session) - arns, err := sdk.ListTasks(bgContext, t.Name(), t.Name()) + api := sdk.NewAPI(session) + arns, err := api.ListTasks(bgContext, t.Name(), t.Name()) assert.NilError(t, err) - tasks, err := sdk.DescribeTasks(bgContext, t.Name(), arns...) - publicIps, err := sdk.GetPublicIPs(context.Background(), tasks[0].NetworkInterface) + tasks, err := api.DescribeTasks(bgContext, t.Name(), arns...) + publicIps, err := api.GetPublicIPs(context.Background(), tasks[0].NetworkInterface) assert.NilError(t, err) for _, ip := range publicIps { icmd.RunCommand("curl", "-I", "http://"+ip).Assert(t, icmd.Success) From 934e7ab9ea2d31df33aa2859ef369703b5ffef9d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 25 Jun 2020 08:14:54 +0200 Subject: [PATCH 134/205] don't set service `Name` so they can be updated by CloudFormation Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 4 +- ecs/go.mod | 2 +- ecs/go.sum | 4 ++ ecs/pkg/amazon/backend/backend.go | 6 --- ecs/pkg/amazon/backend/cloudformation.go | 35 +++++++++++------ ecs/pkg/amazon/backend/down.go | 4 +- ecs/pkg/amazon/backend/list.go | 16 +++++--- ecs/pkg/amazon/backend/up.go | 7 +++- ecs/pkg/amazon/backend/wait.go | 5 +-- ecs/pkg/amazon/sdk/api.go | 2 +- ecs/pkg/amazon/sdk/api_mock.go | 14 +++---- ecs/pkg/amazon/sdk/sdk.go | 50 ++++++++++++++++++------ ecs/pkg/compose/tags.go | 7 ++++ 13 files changed, 103 insertions(+), 53 deletions(-) create mode 100644 ecs/pkg/compose/tags.go diff --git a/ecs/Dockerfile b/ecs/Dockerfile index dead5829..2d2213bd 100644 --- a/ecs/Dockerfile +++ b/ecs/Dockerfile @@ -9,7 +9,8 @@ ENV GO111MODULE=on ARG ALPINE_PKG_DOCKER_VERSION RUN apk add --no-cache \ docker=${ALPINE_PKG_DOCKER_VERSION} \ - make + make \ + build-base COPY go.* . RUN --mount=type=cache,target=/go/pkg/mod \ go mod download @@ -18,7 +19,6 @@ COPY . . FROM base AS make-plugin ARG TARGETOS ARG TARGETARCH -RUN apk add build-base RUN GO111MODULE=on go get github.com/golang/mock/mockgen@latest RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ diff --git a/ecs/go.mod b/ecs/go.mod index 438cf8b0..16f9c9e3 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -14,7 +14,7 @@ require ( github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a + github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 diff --git a/ecs/go.sum b/ecs/go.sum index 9edabc47..5e211f53 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -60,6 +60,10 @@ github.com/compose-spec/compose-go v0.0.0-20200617133919-fca3bb55c5cc h1:jZfF+Hz github.com/compose-spec/compose-go v0.0.0-20200617133919-fca3bb55c5cc/go.mod h1:d3Vb4tH01Pr4YKD3RvfwguRcezDBUYJTVYgpCSRYSVg= github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a h1:FmEuebUePUA0Kd/NSiCmdPG/n6eKdZdBtIbfejVtRS8= github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= +github.com/compose-spec/compose-go v0.0.0-20200624090650-5d46d553c1e6 h1:9rsA2PlPOv50IOnzSiTqCWrWr3u2q7shPr76Y5hlxF0= +github.com/compose-spec/compose-go v0.0.0-20200624090650-5d46d553c1e6/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= +github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 h1:sVvKsoXizFOuJNc8dM91IeET2/zDNFj3hwHgk437iJ8= +github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= diff --git a/ecs/pkg/amazon/backend/backend.go b/ecs/pkg/amazon/backend/backend.go index 0d2f07f9..07764df3 100644 --- a/ecs/pkg/amazon/backend/backend.go +++ b/ecs/pkg/amazon/backend/backend.go @@ -6,12 +6,6 @@ import ( "github.com/docker/ecs-plugin/pkg/amazon/sdk" ) -const ( - ProjectTag = "com.docker.compose.project" - NetworkTag = "com.docker.compose.network" - ServiceTag = "com.docker.compose.service" -) - func NewBackend(profile string, cluster string, region string) (*Backend, error) { sess, err := session.NewSessionWithOptions(session.Options{ Profile: profile, diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index f4111240..98b9b00b 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -31,13 +31,22 @@ const ( ) type FargateCompatibilityChecker struct { - *compatibility.AllowList + compatibility.AllowList +} + +func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortConfig) { + if p.Published == 0 { + p.Published = p.Target + } + if p.Published != p.Target { + c.Error("published port can't be set to a distinct value than container port") + } } // Convert a compose project into a CloudFormation template func (b Backend) Convert(project *types.Project) (*cloudformation.Template, error) { - var checker compatibility.Checker = FargateCompatibilityChecker{ - &compatibility.AllowList{ + var checker compatibility.Checker = &FargateCompatibilityChecker{ + compatibility.AllowList{ Supported: []string{ "services.command", "services.container_name", @@ -161,7 +170,7 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro dependsOn = append(dependsOn, listenerName) serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ ContainerName: service.Name, - ContainerPort: int(port.Published), + ContainerPort: int(port.Target), TargetGroupArn: cloudformation.Ref(targetGroupName), }) } @@ -195,11 +204,11 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro ServiceRegistries: []ecs.Service_ServiceRegistry{serviceRegistry}, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, { - Key: ServiceTag, + Key: compose.ServiceTag, Value: service.Name, }, }, @@ -252,7 +261,7 @@ func createLoadBalancer(project *types.Project, template *cloudformation.Templat }, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, }, @@ -267,7 +276,7 @@ func createListener(service types.ServiceConfig, port types.ServicePortConfig, t "%s%s%dListener", normalizeResourceName(service.Name), strings.ToUpper(port.Protocol), - port.Published, + port.Target, ) //add listener to dependsOn //https://stackoverflow.com/questions/53971873/the-target-group-does-not-have-an-associated-load-balancer @@ -286,7 +295,7 @@ func createListener(service types.ServiceConfig, port types.ServicePortConfig, t }, LoadBalancerArn: loadBalancerARN, Protocol: protocol, - Port: int(port.Published), + Port: int(port.Target), } return listenerName } @@ -304,7 +313,7 @@ func createTargetGroup(project *types.Project, service types.ServiceConfig, port Protocol: protocol, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, }, @@ -371,7 +380,7 @@ func createCluster(project *types.Project, template *cloudformation.Template) st ClusterName: project.Name, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, }, @@ -420,11 +429,11 @@ func convertNetwork(project *types.Project, net types.NetworkConfig, vpc string, VpcId: vpc, Tags: []tags.Tag{ { - Key: ProjectTag, + Key: compose.ProjectTag, Value: project.Name, }, { - Key: NetworkTag, + Key: compose.NetworkTag, Value: net.Name, }, }, diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index 18b4cbf9..fcfbc9ac 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -5,6 +5,7 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/console" ) func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { @@ -22,7 +23,8 @@ func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { return err } - err = b.WaitStackCompletion(ctx, name, compose.StackDelete) + w := console.NewProgressWriter() + err = b.WaitStackCompletion(ctx, name, compose.StackDelete, w) if err != nil { return err } diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index c13d2f90..1193d1fc 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -14,18 +14,22 @@ func (b *Backend) Ps(ctx context.Context, project *types.Project) ([]compose.Ser cluster = project.Name } - status := []compose.ServiceStatus{} - for _, service := range project.Services { - desc, err := b.api.DescribeService(ctx, cluster, service.Name) + status, err := b.api.DescribeServices(ctx, cluster, project.Name) + if err != nil { + return nil, err + } + + for i, state := range status { + s, err := project.GetService(state.Name) if err != nil { return nil, err } ports := []string{} - for _, p := range service.Ports { + for _, p := range s.Ports { ports = append(ports, fmt.Sprintf("*:%d->%d/%s", p.Published, p.Target, p.Protocol)) } - desc.Ports = ports - status = append(status, desc) + state.Ports = ports + status[i] = state } return status, nil } diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 24c75d5c..fae330e6 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -7,6 +7,7 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/console" ) func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { @@ -67,7 +68,11 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { } fmt.Println() - return b.WaitStackCompletion(ctx, project.Name, compose.StackCreate) + w := console.NewProgressWriter() + for k := range template.Resources { + w.ResourceEvent(k, "PENDING", "") + } + return b.WaitStackCompletion(ctx, project.Name, compose.StackCreate, w) } func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, error) { diff --git a/ecs/pkg/amazon/backend/wait.go b/ecs/pkg/amazon/backend/wait.go index 77ad844a..babcaaf6 100644 --- a/ecs/pkg/amazon/backend/wait.go +++ b/ecs/pkg/amazon/backend/wait.go @@ -11,8 +11,7 @@ import ( "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int) error { - w := console.NewProgressWriter() +func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int, w console.ProgressWriter) error { knownEvents := map[string]struct{}{} // Get the unique Stack ID so we can collect events without getting some from previous deployments with same name @@ -53,7 +52,7 @@ func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operatio } knownEvents[*event.EventId] = struct{}{} - resource := fmt.Sprintf("%s %q", aws.StringValue(event.ResourceType), aws.StringValue(event.LogicalResourceId)) + resource := aws.StringValue(event.LogicalResourceId) reason := aws.StringValue(event.ResourceStatusReason) status := aws.StringValue(event.ResourceStatus) w.ResourceEvent(resource, status, reason) diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index 1ad68afa..39e27d37 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -49,7 +49,7 @@ type secretsAPI interface { } type listAPI interface { - DescribeService(ctx context.Context, cluster string, name string) (compose.ServiceStatus, error) + DescribeServices(ctx context.Context, cluster string, project string) ([]compose.ServiceStatus, error) } type waitAPI interface { diff --git a/ecs/pkg/amazon/sdk/api_mock.go b/ecs/pkg/amazon/sdk/api_mock.go index dda73b71..22294ff0 100644 --- a/ecs/pkg/amazon/sdk/api_mock.go +++ b/ecs/pkg/amazon/sdk/api_mock.go @@ -122,19 +122,19 @@ func (mr *MockAPIMockRecorder) DeleteStack(arg0, arg1 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), arg0, arg1) } -// DescribeService mocks base method -func (m *MockAPI) DescribeService(arg0 context.Context, arg1, arg2 string) (compose.ServiceStatus, error) { +// DescribeServices mocks base method +func (m *MockAPI) DescribeServices(arg0 context.Context, arg1, arg2 string) ([]compose.ServiceStatus, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeService", arg0, arg1, arg2) - ret0, _ := ret[0].(compose.ServiceStatus) + ret := m.ctrl.Call(m, "DescribeServices", arg0, arg1, arg2) + ret0, _ := ret[0].([]compose.ServiceStatus) ret1, _ := ret[1].(error) return ret0, ret1 } -// DescribeService indicates an expected call of DescribeService -func (mr *MockAPIMockRecorder) DescribeService(arg0, arg1, arg2 interface{}) *gomock.Call { +// DescribeServices indicates an expected call of DescribeServices +func (mr *MockAPIMockRecorder) DescribeServices(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeService", reflect.TypeOf((*MockAPI)(nil).DescribeService), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeServices", reflect.TypeOf((*MockAPI)(nil).DescribeServices), arg0, arg1, arg2) } // DescribeStackEvents mocks base method diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 3cc2c0fa..1710c314 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -175,7 +175,7 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template StackName: aws.String(name), TemplateBody: aws.String(string(json)), Parameters: param, - TimeoutInMinutes: aws.Int64(10), + TimeoutInMinutes: aws.Int64(15), Capabilities: []*string{ aws.String(cloudformation.CapabilityCapabilityIam), }, @@ -341,20 +341,46 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer compose.LogConsu } } -func (s sdk) DescribeService(ctx context.Context, cluster string, name string) (compose.ServiceStatus, error) { - services, err := s.ECS.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{ - Cluster: aws.String(cluster), - Services: aws.StringSlice([]string{name}), +func (s sdk) DescribeServices(ctx context.Context, cluster string, project string) ([]compose.ServiceStatus, error) { + // TODO handle pagination + list, err := s.ECS.ListServicesWithContext(ctx, &ecs.ListServicesInput{ + Cluster: aws.String(cluster), }) if err != nil { - return compose.ServiceStatus{}, err + return nil, err } - return compose.ServiceStatus{ - ID: *services.Services[0].ServiceName, - Name: name, - Replicas: int(*services.Services[0].RunningCount), - Desired: int(*services.Services[0].DesiredCount), - }, nil + + services, err := s.ECS.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{ + Cluster: aws.String(cluster), + Services: list.ServiceArns, + }) + if err != nil { + return nil, err + } + status := []compose.ServiceStatus{} + for _, service := range services.Services { + var name string + var stack string + for _, t := range service.Tags { + switch *t.Key { + case compose.ProjectTag: + stack = *t.Value + case compose.ServiceTag: + name = *t.Value + } + } + if stack != project { + continue + } + status = append(status, compose.ServiceStatus{ + ID: *service.ServiceName, + Name: name, + Replicas: int(*services.Services[0].RunningCount), + Desired: int(*services.Services[0].DesiredCount), + }) + } + + return status, nil } func (s sdk) ListTasks(ctx context.Context, cluster string, family string) ([]string, error) { diff --git a/ecs/pkg/compose/tags.go b/ecs/pkg/compose/tags.go new file mode 100644 index 00000000..43236d45 --- /dev/null +++ b/ecs/pkg/compose/tags.go @@ -0,0 +1,7 @@ +package compose + +const ( + ProjectTag = "com.docker.compose.project" + NetworkTag = "com.docker.compose.network" + ServiceTag = "com.docker.compose.service" +) From d2911c1ea9c463603e3ea82ea5dbcf3ef62a6835 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 29 Jun 2020 10:44:40 +0200 Subject: [PATCH 135/205] includes:"TAGS" is required for DescribeServices Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/sdk/sdk.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 1710c314..10ad4b5b 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -353,6 +353,7 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, project strin services, err := s.ECS.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{ Cluster: aws.String(cluster), Services: list.ServiceArns, + Include: aws.StringSlice([]string{"TAGS"}), }) if err != nil { return nil, err @@ -375,8 +376,8 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, project strin status = append(status, compose.ServiceStatus{ ID: *service.ServiceName, Name: name, - Replicas: int(*services.Services[0].RunningCount), - Desired: int(*services.Services[0].DesiredCount), + Replicas: int(*service.RunningCount), + Desired: int(*service.DesiredCount), }) } From e2c903c85fde8821f20259cd7978675564bd562d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 30 Jun 2020 11:37:03 +0200 Subject: [PATCH 136/205] Set version by most recent Tag Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 7 ++++++- ecs/Makefile | 12 +++++++++++- ecs/builder.Makefile | 4 +++- ecs/cmd/commands/version.go | 9 +++++++-- ecs/tests/e2e_deploy_services_test.go | 3 +-- ecs/tests/version_test.go | 18 ++++++++++++++++++ 6 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 ecs/tests/version_test.go diff --git a/ecs/Dockerfile b/ecs/Dockerfile index 2d2213bd..9d2b0751 100644 --- a/ecs/Dockerfile +++ b/ecs/Dockerfile @@ -14,12 +14,14 @@ RUN apk add --no-cache \ COPY go.* . RUN --mount=type=cache,target=/go/pkg/mod \ go mod download -COPY . . FROM base AS make-plugin ARG TARGETOS ARG TARGETARCH RUN GO111MODULE=on go get github.com/golang/mock/mockgen@latest +ARG COMMIT +ARG TAG +COPY . . RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ GOOS=${TARGETOS} \ @@ -27,6 +29,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ make -f builder.Makefile build FROM base AS make-cross +ARG COMMIT +ARG TAG +COPY . . RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ make -f builder.Makefile cross diff --git a/ecs/Makefile b/ecs/Makefile index 22dd3604..cf9c48cc 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -3,21 +3,31 @@ PWD=$(shell pwd) export DOCKER_BUILDKIT=1 +COMMIT := $(shell git rev-parse --short HEAD) +TAG := $(shell git describe --tags --dirty --match "v*") + .DEFAULT_GOAL := build build: ## Build for the current @docker build . \ --output ./dist \ --platform ${PLATFORM} \ + --build-arg COMMIT=${COMMIT} \ + --build-arg TAG=${TAG} \ --target build cross: ## Cross build for linux, macos and windows @docker build . \ --output ./dist \ + --build-arg COMMIT=${COMMIT} \ + --build-arg TAG=${TAG} \ --target cross test: build ## Run tests - @docker build . --target test + @docker build . \ + --build-arg COMMIT=${COMMIT} \ + --build-arg TAG=${TAG} \ + --target test e2e: build ## Run tests go test ./... -v -tags=e2e diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile index 18bd2da2..4bdf2789 100644 --- a/ecs/builder.Makefile +++ b/ecs/builder.Makefile @@ -7,7 +7,9 @@ ifeq ($(GOOS),windows) endif STATIC_FLAGS=CGO_ENABLED=0 -LDFLAGS:="-s -w" +LDFLAGS := "-s -w \ + -X github.com/docker/ecs-plugin/cmd/commands.GitCommit=$(COMMIT) \ + -X github.com/docker/ecs-plugin/cmd/commands.Version=$(TAG)" GO_BUILD=$(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) BINARY=dist/docker-ecs diff --git a/ecs/cmd/commands/version.go b/ecs/cmd/commands/version.go index d3b44687..58e7ccab 100644 --- a/ecs/cmd/commands/version.go +++ b/ecs/cmd/commands/version.go @@ -6,14 +6,19 @@ import ( "github.com/spf13/cobra" ) -const Version = "0.0.1" +var ( + // Version is the git tag that this was built from. + Version = "unknown" + // GitCommit is the commit that this was built from. + GitCommit = "unknown" +) func VersionCommand() *cobra.Command { return &cobra.Command{ Use: "version", Short: "Show version.", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Fprintf(cmd.OutOrStdout(), "Docker ECS plugin %s\n", Version) + fmt.Fprintf(cmd.OutOrStdout(), "Docker ECS plugin %s (%s)\n", Version, GitCommit) return nil }, } diff --git a/ecs/tests/e2e_deploy_services_test.go b/ecs/tests/e2e_deploy_services_test.go index e55fc93b..32b6dab5 100644 --- a/ecs/tests/e2e_deploy_services_test.go +++ b/ecs/tests/e2e_deploy_services_test.go @@ -6,10 +6,9 @@ import ( "context" "testing" - "github.com/docker/ecs-plugin/pkg/amazon/sdk" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" + "github.com/docker/ecs-plugin/pkg/amazon/sdk" "github.com/docker/ecs-plugin/pkg/docker" "gotest.tools/v3/assert" "gotest.tools/v3/fs" diff --git a/ecs/tests/version_test.go b/ecs/tests/version_test.go new file mode 100644 index 00000000..ad4d6455 --- /dev/null +++ b/ecs/tests/version_test.go @@ -0,0 +1,18 @@ +package tests + +import ( + "strings" + "testing" + + "gotest.tools/v3/assert" + "gotest.tools/v3/icmd" +) + +func TestVersionIsSet(t *testing.T) { + cmd, cleanup, _ := dockerCli.createTestCmd() + defer cleanup() + + cmd.Command = dockerCli.Command("ecs", "version") + out := icmd.RunCmd(cmd).Assert(t, icmd.Success).Stdout() + assert.Check(t, !strings.Contains(out, "unknown")) +} From 2bc1b710f2b11fe5b584815a638be92b87e708ab Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 30 Jun 2020 15:52:04 +0200 Subject: [PATCH 137/205] Testcase to check resources get tagged Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation_test.go | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 001931b0..fec16720 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -2,8 +2,11 @@ package backend import ( "fmt" + "reflect" "testing" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" @@ -105,6 +108,31 @@ services: assert.Check(t, lb.Type == elbv2.LoadBalancerTypeEnumNetwork) } +func TestResourcesHaveProjectTagSet(t *testing.T) { + template := convertYaml(t, ` +version: "3" +services: + test: + image: nginx + ports: + - 80:80 + - 88:88 +`) + for _, r := range template.Resources { + tags := reflect.Indirect(reflect.ValueOf(r)).FieldByName("Tags") + if !tags.IsValid() { + continue + } + for i := 0; i < tags.Len(); i++ { + k := tags.Index(i).FieldByName("Key").String() + v := tags.Index(i).FieldByName("Value").String() + if k == compose.ProjectTag { + assert.Equal(t, v, "Test") + } + } + } +} + func convertResultAsString(t *testing.T, project *types.Project, clusterName string) string { client, err := NewBackend("", clusterName, "") assert.NilError(t, err) From c0c31de0c8fd72b6b7bd69cbcbb9d3051d4ce2d7 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 30 Jun 2020 17:15:52 +0200 Subject: [PATCH 138/205] testcase to check service mapping Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 15 +++++++ ecs/pkg/amazon/backend/cloudformation_test.go | 44 ++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 98b9b00b..3df5b8ab 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -43,6 +43,19 @@ func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortCo } } +func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) { + add := []string{} + for _, cap := range service.CapAdd { + switch cap { + case "SYS_PTRACE": + add = append(add, cap) + default: + c.Error("service.cap_add = %s", cap) + } + } + service.CapAdd = add +} + // Convert a compose project into a CloudFormation template func (b Backend) Convert(project *types.Project) (*cloudformation.Template, error) { var checker compatibility.Checker = &FargateCompatibilityChecker{ @@ -50,9 +63,11 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro Supported: []string{ "services.command", "services.container_name", + "services.cap_drop", "services.depends_on", "services.entrypoint", "services.environment", + "services.init", "services.healthcheck", "services.healthcheck.interval", "services.healthcheck.start_period", diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index fec16720..2fcd1542 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -5,16 +5,16 @@ import ( "reflect" "testing" - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/aws/aws-sdk-go/service/elbv2" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" + "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" + "github.com/docker/ecs-plugin/pkg/compose" "gotest.tools/v3/assert" "gotest.tools/v3/golden" ) @@ -108,6 +108,46 @@ services: assert.Check(t, lb.Type == elbv2.LoadBalancerTypeEnumNetwork) } +func TestServiceMapping(t *testing.T) { + template := convertYaml(t, ` +version: "3" +services: + test: + image: "image" + command: "command" + entrypoint: "entrypoint" + environment: + - "FOO=BAR" + cap_add: + - SYS_PTRACE + cap_drop: + - SYSLOG + init: true + user: "user" + working_dir: "working_dir" +`) + def := template.Resources["TestTaskDefinition"].(*ecs.TaskDefinition) + container := def.ContainerDefinitions[0] + assert.Equal(t, container.Image, "docker.io/library/image") + assert.Equal(t, container.Command[0], "command") + assert.Equal(t, container.EntryPoint[0], "entrypoint") + assert.Equal(t, get(container.Environment, "FOO"), "BAR") + assert.Check(t, container.LinuxParameters.InitProcessEnabled) + assert.Equal(t, container.LinuxParameters.Capabilities.Add[0], "SYS_PTRACE") + assert.Equal(t, container.LinuxParameters.Capabilities.Drop[0], "SYSLOG") + assert.Equal(t, container.User, "user") + assert.Equal(t, container.WorkingDirectory, "working_dir") +} + +func get(l []ecs.TaskDefinition_KeyValuePair, name string) string { + for _, e := range l { + if e.Name == name { + return e.Value + } + } + return "" +} + func TestResourcesHaveProjectTagSet(t *testing.T) { template := convertYaml(t, ` version: "3" From 5c53138a34fcd75f825c3b38c0b243245c2ea619 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 1 Jul 2020 16:21:15 +0200 Subject: [PATCH 139/205] Drop use of mockgen made obsolete as we use CloudFormation Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 1 - ecs/builder.Makefile | 5 +- ecs/go.mod | 1 - ecs/go.sum | 16 -- ecs/pkg/amazon/backend/down_test.go | 33 --- ecs/pkg/amazon/sdk/api.go | 36 +--- ecs/pkg/amazon/sdk/api_mock.go | 316 ---------------------------- ecs/pkg/amazon/sdk/sdk.go | 12 -- 8 files changed, 7 insertions(+), 413 deletions(-) delete mode 100644 ecs/pkg/amazon/backend/down_test.go delete mode 100644 ecs/pkg/amazon/sdk/api_mock.go diff --git a/ecs/Dockerfile b/ecs/Dockerfile index 9d2b0751..185a080e 100644 --- a/ecs/Dockerfile +++ b/ecs/Dockerfile @@ -18,7 +18,6 @@ RUN --mount=type=cache,target=/go/pkg/mod \ FROM base AS make-plugin ARG TARGETOS ARG TARGETARCH -RUN GO111MODULE=on go get github.com/golang/mock/mockgen@latest ARG COMMIT ARG TAG COPY . . diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile index 4bdf2789..a3954991 100644 --- a/ecs/builder.Makefile +++ b/ecs/builder.Makefile @@ -22,10 +22,7 @@ all: build clean: rm -rf dist/ -generate: pkg/amazon/sdk/api_mock.go - go generate ./... - -build: generate +build: $(GO_BUILD) -v -o $(BINARY_WITH_EXTENSION) cmd/main/main.go cross: diff --git a/ecs/go.mod b/ecs/go.mod index 16f9c9e3..163b56fc 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -26,7 +26,6 @@ require ( github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.1 // indirect - github.com/golang/mock v1.4.3 github.com/gorilla/mux v1.7.3 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/jinzhu/gorm v1.9.12 // indirect diff --git a/ecs/go.sum b/ecs/go.sum index 5e211f53..27588bfe 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -54,14 +54,6 @@ github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiK github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo= github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= -github.com/compose-spec/compose-go v0.0.0-20200616184722-5b8dc203fd7f h1:XE6hHZdPjxN8uGaRlvdCB8YwXbz1PXnQ0CboNygdL2o= -github.com/compose-spec/compose-go v0.0.0-20200616184722-5b8dc203fd7f/go.mod h1:d3Vb4tH01Pr4YKD3RvfwguRcezDBUYJTVYgpCSRYSVg= -github.com/compose-spec/compose-go v0.0.0-20200617133919-fca3bb55c5cc h1:jZfF+HzxW+c8Em308MvcK7j5+ZqIAWqFjN1RZnVFzck= -github.com/compose-spec/compose-go v0.0.0-20200617133919-fca3bb55c5cc/go.mod h1:d3Vb4tH01Pr4YKD3RvfwguRcezDBUYJTVYgpCSRYSVg= -github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a h1:FmEuebUePUA0Kd/NSiCmdPG/n6eKdZdBtIbfejVtRS8= -github.com/compose-spec/compose-go v0.0.0-20200622094647-0bb9a6c7d89a/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= -github.com/compose-spec/compose-go v0.0.0-20200624090650-5d46d553c1e6 h1:9rsA2PlPOv50IOnzSiTqCWrWr3u2q7shPr76Y5hlxF0= -github.com/compose-spec/compose-go v0.0.0-20200624090650-5d46d553c1e6/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 h1:sVvKsoXizFOuJNc8dM91IeET2/zDNFj3hwHgk437iJ8= github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= @@ -133,8 +125,6 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -144,8 +134,6 @@ github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -412,7 +400,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -426,7 +413,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -474,7 +460,5 @@ gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/ecs/pkg/amazon/backend/down_test.go b/ecs/pkg/amazon/backend/down_test.go deleted file mode 100644 index 7439f626..00000000 --- a/ecs/pkg/amazon/backend/down_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package backend - -import ( - "context" - "testing" - - "github.com/compose-spec/compose-go/cli" - "github.com/docker/ecs-plugin/pkg/amazon/sdk" - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/golang/mock/gomock" -) - -func TestDown(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - m := sdk.NewMockAPI(ctrl) - c := &Backend{ - Cluster: "test_cluster", - Region: "region", - api: m, - } - ctx := context.TODO() - recorder := m.EXPECT() - recorder.DeleteStack(ctx, "test_project").Return(nil) - recorder.GetStackID(ctx, "test_project").Return("stack-123", nil) - recorder.WaitStackComplete(ctx, "stack-123", compose.StackDelete).Return(nil) - recorder.DescribeStackEvents(ctx, "stack-123").Return(nil, nil) - - c.Down(ctx, cli.ProjectOptions{ - ConfigPaths: []string{}, - Name: "test_project", - }) -} diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index 39e27d37..38782f5d 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -8,52 +8,28 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) -//go:generate mockgen -destination=./api_mock.go -self_package "github.com/docker/ecs-plugin/pkg/amazon/sdk" -package=sdk . API - type API interface { - downAPI - upAPI - logsAPI - secretsAPI - listAPI -} - -type upAPI interface { - waitAPI GetDefaultVPC(ctx context.Context) (string, error) VpcExists(ctx context.Context, vpcID string) (bool, error) GetSubNets(ctx context.Context, vpcID string) ([]string, error) - ClusterExists(ctx context.Context, name string) (bool, error) StackExists(ctx context.Context, name string) (bool, error) CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error + DeleteStack(ctx context.Context, name string) error + DescribeServices(ctx context.Context, cluster string, project string) ([]compose.ServiceStatus, error) + GetStackID(ctx context.Context, name string) (string, error) + WaitStackComplete(ctx context.Context, name string, operation int) error + DescribeStackEvents(ctx context.Context, stackID string) ([]*cf.StackEvent, error) LoadBalancerExists(ctx context.Context, name string) (bool, error) GetLoadBalancerARN(ctx context.Context, name string) (string, error) -} -type downAPI interface { - DeleteStack(ctx context.Context, name string) error - DeleteCluster(ctx context.Context, name string) error -} + ClusterExists(ctx context.Context, name string) (bool, error) -type logsAPI interface { GetLogs(ctx context.Context, name string, consumer compose.LogConsumer) error -} -type secretsAPI interface { CreateSecret(ctx context.Context, secret compose.Secret) (string, error) InspectSecret(ctx context.Context, id string) (compose.Secret, error) ListSecrets(ctx context.Context) ([]compose.Secret, error) DeleteSecret(ctx context.Context, id string, recover bool) error } - -type listAPI interface { - DescribeServices(ctx context.Context, cluster string, project string) ([]compose.ServiceStatus, error) -} - -type waitAPI interface { - GetStackID(ctx context.Context, name string) (string, error) - WaitStackComplete(ctx context.Context, name string, operation int) error - DescribeStackEvents(ctx context.Context, stackID string) ([]*cf.StackEvent, error) -} diff --git a/ecs/pkg/amazon/sdk/api_mock.go b/ecs/pkg/amazon/sdk/api_mock.go deleted file mode 100644 index 22294ff0..00000000 --- a/ecs/pkg/amazon/sdk/api_mock.go +++ /dev/null @@ -1,316 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/docker/ecs-plugin/pkg/amazon/sdk (interfaces: API) - -// Package sdk is a generated GoMock package. -package sdk - -import ( - context "context" - cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" - cloudformation0 "github.com/awslabs/goformation/v4/cloudformation" - compose "github.com/docker/ecs-plugin/pkg/compose" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockAPI is a mock of API interface -type MockAPI struct { - ctrl *gomock.Controller - recorder *MockAPIMockRecorder -} - -// MockAPIMockRecorder is the mock recorder for MockAPI -type MockAPIMockRecorder struct { - mock *MockAPI -} - -// NewMockAPI creates a new mock instance -func NewMockAPI(ctrl *gomock.Controller) *MockAPI { - mock := &MockAPI{ctrl: ctrl} - mock.recorder = &MockAPIMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockAPI) EXPECT() *MockAPIMockRecorder { - return m.recorder -} - -// ClusterExists mocks base method -func (m *MockAPI) ClusterExists(arg0 context.Context, arg1 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClusterExists", arg0, arg1) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ClusterExists indicates an expected call of ClusterExists -func (mr *MockAPIMockRecorder) ClusterExists(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), arg0, arg1) -} - -// CreateSecret mocks base method -func (m *MockAPI) CreateSecret(arg0 context.Context, arg1 compose.Secret) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateSecret", arg0, arg1) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateSecret indicates an expected call of CreateSecret -func (mr *MockAPIMockRecorder) CreateSecret(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSecret", reflect.TypeOf((*MockAPI)(nil).CreateSecret), arg0, arg1) -} - -// CreateStack mocks base method -func (m *MockAPI) CreateStack(arg0 context.Context, arg1 string, arg2 *cloudformation0.Template, arg3 map[string]string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateStack", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateStack indicates an expected call of CreateStack -func (mr *MockAPIMockRecorder) CreateStack(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateStack", reflect.TypeOf((*MockAPI)(nil).CreateStack), arg0, arg1, arg2, arg3) -} - -// DeleteCluster mocks base method -func (m *MockAPI) DeleteCluster(arg0 context.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteCluster", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteCluster indicates an expected call of DeleteCluster -func (mr *MockAPIMockRecorder) DeleteCluster(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCluster", reflect.TypeOf((*MockAPI)(nil).DeleteCluster), arg0, arg1) -} - -// DeleteSecret mocks base method -func (m *MockAPI) DeleteSecret(arg0 context.Context, arg1 string, arg2 bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSecret", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteSecret indicates an expected call of DeleteSecret -func (mr *MockAPIMockRecorder) DeleteSecret(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSecret", reflect.TypeOf((*MockAPI)(nil).DeleteSecret), arg0, arg1, arg2) -} - -// DeleteStack mocks base method -func (m *MockAPI) DeleteStack(arg0 context.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteStack", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteStack indicates an expected call of DeleteStack -func (mr *MockAPIMockRecorder) DeleteStack(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStack", reflect.TypeOf((*MockAPI)(nil).DeleteStack), arg0, arg1) -} - -// DescribeServices mocks base method -func (m *MockAPI) DescribeServices(arg0 context.Context, arg1, arg2 string) ([]compose.ServiceStatus, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeServices", arg0, arg1, arg2) - ret0, _ := ret[0].([]compose.ServiceStatus) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DescribeServices indicates an expected call of DescribeServices -func (mr *MockAPIMockRecorder) DescribeServices(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeServices", reflect.TypeOf((*MockAPI)(nil).DescribeServices), arg0, arg1, arg2) -} - -// DescribeStackEvents mocks base method -func (m *MockAPI) DescribeStackEvents(arg0 context.Context, arg1 string) ([]*cloudformation.StackEvent, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DescribeStackEvents", arg0, arg1) - ret0, _ := ret[0].([]*cloudformation.StackEvent) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DescribeStackEvents indicates an expected call of DescribeStackEvents -func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0, arg1) -} - -// GetDefaultVPC mocks base method -func (m *MockAPI) GetDefaultVPC(arg0 context.Context) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetDefaultVPC", arg0) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetDefaultVPC indicates an expected call of GetDefaultVPC -func (mr *MockAPIMockRecorder) GetDefaultVPC(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDefaultVPC", reflect.TypeOf((*MockAPI)(nil).GetDefaultVPC), arg0) -} - -// GetLoadBalancerARN mocks base method -func (m *MockAPI) GetLoadBalancerARN(arg0 context.Context, arg1 string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLoadBalancerARN", arg0, arg1) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetLoadBalancerARN indicates an expected call of GetLoadBalancerARN -func (mr *MockAPIMockRecorder) GetLoadBalancerARN(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoadBalancerARN", reflect.TypeOf((*MockAPI)(nil).GetLoadBalancerARN), arg0, arg1) -} - -// GetLogs mocks base method -func (m *MockAPI) GetLogs(arg0 context.Context, arg1 string, arg2 compose.LogConsumer) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLogs", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// GetLogs indicates an expected call of GetLogs -func (mr *MockAPIMockRecorder) GetLogs(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogs", reflect.TypeOf((*MockAPI)(nil).GetLogs), arg0, arg1, arg2) -} - -// GetStackID mocks base method -func (m *MockAPI) GetStackID(arg0 context.Context, arg1 string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetStackID", arg0, arg1) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetStackID indicates an expected call of GetStackID -func (mr *MockAPIMockRecorder) GetStackID(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStackID", reflect.TypeOf((*MockAPI)(nil).GetStackID), arg0, arg1) -} - -// GetSubNets mocks base method -func (m *MockAPI) GetSubNets(arg0 context.Context, arg1 string) ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSubNets", arg0, arg1) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetSubNets indicates an expected call of GetSubNets -func (mr *MockAPIMockRecorder) GetSubNets(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubNets", reflect.TypeOf((*MockAPI)(nil).GetSubNets), arg0, arg1) -} - -// InspectSecret mocks base method -func (m *MockAPI) InspectSecret(arg0 context.Context, arg1 string) (compose.Secret, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InspectSecret", arg0, arg1) - ret0, _ := ret[0].(compose.Secret) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// InspectSecret indicates an expected call of InspectSecret -func (mr *MockAPIMockRecorder) InspectSecret(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InspectSecret", reflect.TypeOf((*MockAPI)(nil).InspectSecret), arg0, arg1) -} - -// ListSecrets mocks base method -func (m *MockAPI) ListSecrets(arg0 context.Context) ([]compose.Secret, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListSecrets", arg0) - ret0, _ := ret[0].([]compose.Secret) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListSecrets indicates an expected call of ListSecrets -func (mr *MockAPIMockRecorder) ListSecrets(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSecrets", reflect.TypeOf((*MockAPI)(nil).ListSecrets), arg0) -} - -// LoadBalancerExists mocks base method -func (m *MockAPI) LoadBalancerExists(arg0 context.Context, arg1 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadBalancerExists", arg0, arg1) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// LoadBalancerExists indicates an expected call of LoadBalancerExists -func (mr *MockAPIMockRecorder) LoadBalancerExists(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerExists", reflect.TypeOf((*MockAPI)(nil).LoadBalancerExists), arg0, arg1) -} - -// StackExists mocks base method -func (m *MockAPI) StackExists(arg0 context.Context, arg1 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StackExists", arg0, arg1) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// StackExists indicates an expected call of StackExists -func (mr *MockAPIMockRecorder) StackExists(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StackExists", reflect.TypeOf((*MockAPI)(nil).StackExists), arg0, arg1) -} - -// VpcExists mocks base method -func (m *MockAPI) VpcExists(arg0 context.Context, arg1 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "VpcExists", arg0, arg1) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// VpcExists indicates an expected call of VpcExists -func (mr *MockAPIMockRecorder) VpcExists(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VpcExists", reflect.TypeOf((*MockAPI)(nil).VpcExists), arg0, arg1) -} - -// WaitStackComplete mocks base method -func (m *MockAPI) WaitStackComplete(arg0 context.Context, arg1 string, arg2 int) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WaitStackComplete", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// WaitStackComplete indicates an expected call of WaitStackComplete -func (mr *MockAPIMockRecorder) WaitStackComplete(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitStackComplete", reflect.TypeOf((*MockAPI)(nil).WaitStackComplete), arg0, arg1, arg2) -} diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 10ad4b5b..7359862c 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -70,18 +70,6 @@ func (s sdk) CreateCluster(ctx context.Context, name string) (string, error) { return *response.Cluster.Status, nil } -func (s sdk) DeleteCluster(ctx context.Context, name string) error { - logrus.Debug("Delete cluster ", name) - response, err := s.ECS.DeleteClusterWithContext(ctx, &ecs.DeleteClusterInput{Cluster: aws.String(name)}) - if err != nil { - return err - } - if *response.Cluster.Status == "INACTIVE" { - return nil - } - return fmt.Errorf("Failed to delete cluster, status: %s" + *response.Cluster.Status) -} - func (s sdk) VpcExists(ctx context.Context, vpcID string) (bool, error) { logrus.Debug("Check if VPC exists: ", vpcID) _, err := s.EC2.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{VpcIds: []*string{&vpcID}}) From 2917251f5fdfad551ccb469e31fe4fb41d2d9c25 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 2 Jul 2020 16:51:55 +0200 Subject: [PATCH 140/205] clarify project status Signed-off-by: Nicolas De Loof --- ecs/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ecs/README.md b/ecs/README.md index c0cb834b..a4e0caf5 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -1,5 +1,9 @@ # Docker CLI plugin for Amazon ECS +## Status + +:exclamation: The Docker ECS plugin is still in Beta. It's design and UX will evolve until 1.0 Final release. + ## Architecture ECS plugin is a [Docker CLI plugin](https://docs.docker.com/engine/extend/cli_plugins/) From 324443deb618edfb298fcbab0a74dbd7b83b4333 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 2 Jul 2020 11:43:20 +0200 Subject: [PATCH 141/205] Customize SDK requests to AWS API with user-agent Signed-off-by: Nicolas De Loof --- ecs/builder.Makefile | 4 ++-- ecs/cmd/commands/version.go | 11 +++-------- ecs/cmd/commands/version_test.go | 4 +++- ecs/cmd/main/main.go | 3 ++- ecs/internal/version.go | 8 ++++++++ ecs/pkg/amazon/sdk/sdk.go | 7 +++++++ 6 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 ecs/internal/version.go diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile index a3954991..ae52173d 100644 --- a/ecs/builder.Makefile +++ b/ecs/builder.Makefile @@ -8,8 +8,8 @@ endif STATIC_FLAGS=CGO_ENABLED=0 LDFLAGS := "-s -w \ - -X github.com/docker/ecs-plugin/cmd/commands.GitCommit=$(COMMIT) \ - -X github.com/docker/ecs-plugin/cmd/commands.Version=$(TAG)" + -X github.com/docker/ecs-plugin/internal.GitCommit=$(COMMIT) \ + -X github.com/docker/ecs-plugin/internal.Version=$(TAG)" GO_BUILD=$(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) BINARY=dist/docker-ecs diff --git a/ecs/cmd/commands/version.go b/ecs/cmd/commands/version.go index 58e7ccab..ce025389 100644 --- a/ecs/cmd/commands/version.go +++ b/ecs/cmd/commands/version.go @@ -3,14 +3,9 @@ package commands import ( "fmt" - "github.com/spf13/cobra" -) + "github.com/docker/ecs-plugin/internal" -var ( - // Version is the git tag that this was built from. - Version = "unknown" - // GitCommit is the commit that this was built from. - GitCommit = "unknown" + "github.com/spf13/cobra" ) func VersionCommand() *cobra.Command { @@ -18,7 +13,7 @@ func VersionCommand() *cobra.Command { Use: "version", Short: "Show version.", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Fprintf(cmd.OutOrStdout(), "Docker ECS plugin %s (%s)\n", Version, GitCommit) + fmt.Fprintf(cmd.OutOrStdout(), "Docker ECS plugin %s (%s)\n", internal.Version, internal.GitCommit) return nil }, } diff --git a/ecs/cmd/commands/version_test.go b/ecs/cmd/commands/version_test.go index 4c0ed7e6..43a9ab81 100644 --- a/ecs/cmd/commands/version_test.go +++ b/ecs/cmd/commands/version_test.go @@ -5,6 +5,8 @@ import ( "strings" "testing" + "github.com/docker/ecs-plugin/internal" + "gotest.tools/v3/assert" ) @@ -14,5 +16,5 @@ func TestVersion(t *testing.T) { root.SetOut(&out) root.SetArgs([]string{"version"}) root.Execute() - assert.Check(t, strings.Contains(out.String(), Version)) + assert.Check(t, strings.Contains(out.String(), internal.Version)) } diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go index 5dd1f4dd..f4140394 100644 --- a/ecs/cmd/main/main.go +++ b/ecs/cmd/main/main.go @@ -2,6 +2,7 @@ package main import ( "github.com/docker/ecs-plugin/cmd/commands" + "github.com/docker/ecs-plugin/internal" "github.com/docker/cli/cli-plugins/manager" "github.com/docker/cli/cli-plugins/plugin" @@ -16,7 +17,7 @@ func main() { }, manager.Metadata{ SchemaVersion: "0.1.0", Vendor: "Docker Inc.", - Version: commands.Version, + Version: internal.Version, Experimental: true, }) } diff --git a/ecs/internal/version.go b/ecs/internal/version.go new file mode 100644 index 00000000..b0fc1f6a --- /dev/null +++ b/ecs/internal/version.go @@ -0,0 +1,8 @@ +package internal + +var ( + // Version is the git tag that this was built from. + Version = "unknown" + // GitCommit is the commit that this was built from. + GitCommit = "unknown" +) diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 7359862c..af38523b 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -6,6 +6,10 @@ import ( "strings" "time" + "github.com/docker/ecs-plugin/internal" + + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" @@ -39,6 +43,9 @@ type sdk struct { } func NewAPI(sess *session.Session) API { + sess.Handlers.Build.PushBack(func(r *request.Request) { + request.AddToUserAgent(r, fmt.Sprintf("Docker CLI %s", internal.Version)) + }) return sdk{ ECS: ecs.New(sess), EC2: ec2.New(sess), From f892ee1004fc4cdd9a1d78663b33be36d34a8e18 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 3 Jul 2020 11:07:08 +0200 Subject: [PATCH 142/205] `ps` shows LoadBalancer URL Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/list.go | 28 ++++++++++++++-- ecs/pkg/amazon/backend/up.go | 1 - ecs/pkg/amazon/sdk/api.go | 8 +++-- ecs/pkg/amazon/sdk/sdk.go | 58 +++++++++++++++++++--------------- ecs/pkg/compose/types.go | 7 ++++ 5 files changed, 71 insertions(+), 31 deletions(-) diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index 1193d1fc..00b4adee 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -14,7 +14,31 @@ func (b *Backend) Ps(ctx context.Context, project *types.Project) ([]compose.Ser cluster = project.Name } - status, err := b.api.DescribeServices(ctx, cluster, project.Name) + resources, err := b.api.ListStackResources(ctx, project.Name) + if err != nil { + return nil, err + } + + var loadBalancer string + if lb, ok := project.Extensions[compose.ExtensionLB]; ok { + loadBalancer = lb.(string) + } + servicesARN := []string{} + for _, r := range resources { + switch r.Type { + case "AWS::ECS::Service": + servicesARN = append(servicesARN, r.ARN) + case "AWS::ElasticLoadBalancingV2::LoadBalancer": + loadBalancer = r.ARN + } + } + + status, err := b.api.DescribeServices(ctx, cluster, servicesARN) + if err != nil { + return nil, err + } + + url, err := b.api.GetLoadBalancerURL(ctx, loadBalancer) if err != nil { return nil, err } @@ -26,7 +50,7 @@ func (b *Backend) Ps(ctx context.Context, project *types.Project) ([]compose.Ser } ports := []string{} for _, p := range s.Ports { - ports = append(ports, fmt.Sprintf("*:%d->%d/%s", p.Published, p.Target, p.Protocol)) + ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", url, p.Published, p.Target, p.Protocol)) } state.Ports = ports status[i] = state diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index fae330e6..104fef91 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -105,7 +105,6 @@ func (b Backend) GetLoadBalancer(ctx context.Context, project *types.Project) (s if !ok { return "", fmt.Errorf("Load Balancer does not exist: %s", lb) } - return b.api.GetLoadBalancerARN(ctx, lbName) } return "", nil } diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index 38782f5d..057823c5 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -16,13 +16,15 @@ type API interface { StackExists(ctx context.Context, name string) (bool, error) CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error DeleteStack(ctx context.Context, name string) error - DescribeServices(ctx context.Context, cluster string, project string) ([]compose.ServiceStatus, error) + ListStackResources(ctx context.Context, name string) ([]compose.StackResource, error) GetStackID(ctx context.Context, name string) (string, error) WaitStackComplete(ctx context.Context, name string, operation int) error DescribeStackEvents(ctx context.Context, stackID string) ([]*cf.StackEvent, error) - LoadBalancerExists(ctx context.Context, name string) (bool, error) - GetLoadBalancerARN(ctx context.Context, name string) (string, error) + DescribeServices(ctx context.Context, cluster string, arns []string) ([]compose.ServiceStatus, error) + + LoadBalancerExists(ctx context.Context, arn string) (bool, error) + GetLoadBalancerURL(ctx context.Context, arn string) (string, error) ClusterExists(ctx context.Context, name string) (bool, error) diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index af38523b..5874a800 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -222,6 +222,27 @@ func (s sdk) DescribeStackEvents(ctx context.Context, stackID string) ([]*cloudf } } +func (s sdk) ListStackResources(ctx context.Context, name string) ([]compose.StackResource, error) { + // FIXME handle pagination + res, err := s.CF.ListStackResourcesWithContext(ctx, &cloudformation.ListStackResourcesInput{ + StackName: aws.String(name), + }) + if err != nil { + return nil, err + } + + resources := []compose.StackResource{} + for _, r := range res.StackResourceSummaries { + resources = append(resources, compose.StackResource{ + LogicalID: *r.LogicalResourceId, + Type: *r.ResourceType, + ARN: *r.PhysicalResourceId, + Status: *r.ResourceStatus, + }) + } + return resources, nil +} + func (s sdk) DeleteStack(ctx context.Context, name string) error { logrus.Debug("Delete CloudFormation stack") _, err := s.CF.DeleteStackWithContext(ctx, &cloudformation.DeleteStackInput{ @@ -270,7 +291,6 @@ func (s sdk) InspectSecret(ctx context.Context, id string) (compose.Secret, erro } func (s sdk) ListSecrets(ctx context.Context) ([]compose.Secret, error) { - logrus.Debug("List secrets ...") response, err := s.SM.ListSecrets(&secretsmanager.ListSecretsInput{}) if err != nil { @@ -336,18 +356,10 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer compose.LogConsu } } -func (s sdk) DescribeServices(ctx context.Context, cluster string, project string) ([]compose.ServiceStatus, error) { - // TODO handle pagination - list, err := s.ECS.ListServicesWithContext(ctx, &ecs.ListServicesInput{ - Cluster: aws.String(cluster), - }) - if err != nil { - return nil, err - } - +func (s sdk) DescribeServices(ctx context.Context, cluster string, arns []string) ([]compose.ServiceStatus, error) { services, err := s.ECS.DescribeServicesWithContext(ctx, &ecs.DescribeServicesInput{ Cluster: aws.String(cluster), - Services: list.ServiceArns, + Services: aws.StringSlice(arns), Include: aws.StringSlice([]string{"TAGS"}), }) if err != nil { @@ -356,17 +368,13 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, project strin status := []compose.ServiceStatus{} for _, service := range services.Services { var name string - var stack string for _, t := range service.Tags { - switch *t.Key { - case compose.ProjectTag: - stack = *t.Value - case compose.ServiceTag: + if *t.Key == compose.ServiceTag { name = *t.Value } } - if stack != project { - continue + if name == "" { + return nil, fmt.Errorf("service %s doesn't have a %s tag", *service.ServiceArn, compose.ServiceTag) } status = append(status, compose.ServiceStatus{ ID: *service.ServiceName, @@ -410,10 +418,10 @@ func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string return publicIPs, nil } -func (s sdk) LoadBalancerExists(ctx context.Context, name string) (bool, error) { - logrus.Debug("Check if cluster was already created: ", name) +func (s sdk) LoadBalancerExists(ctx context.Context, arn string) (bool, error) { + logrus.Debug("Check if LoadBalancer exists: ", arn) lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{ - Names: []*string{aws.String(name)}, + LoadBalancerArns: []*string{aws.String(arn)}, }) if err != nil { return false, err @@ -421,13 +429,13 @@ func (s sdk) LoadBalancerExists(ctx context.Context, name string) (bool, error) return len(lbs.LoadBalancers) > 0, nil } -func (s sdk) GetLoadBalancerARN(ctx context.Context, name string) (string, error) { - logrus.Debug("Check if cluster was already created: ", name) +func (s sdk) GetLoadBalancerURL(ctx context.Context, arn string) (string, error) { + logrus.Debug("Retrieve load balancer URL: ", arn) lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{ - Names: []*string{aws.String(name)}, + LoadBalancerArns: []*string{aws.String(arn)}, }) if err != nil { return "", err } - return *lbs.LoadBalancers[0].LoadBalancerArn, nil + return *lbs.LoadBalancers[0].DNSName, nil } diff --git a/ecs/pkg/compose/types.go b/ecs/pkg/compose/types.go index ae56e65e..370bfc4b 100644 --- a/ecs/pkg/compose/types.go +++ b/ecs/pkg/compose/types.go @@ -2,6 +2,13 @@ package compose import "encoding/json" +type StackResource struct { + LogicalID string + Type string + ARN string + Status string +} + type ServiceStatus struct { ID string Name string From 4700fed83660354cc63c804c4399d23c0eef2456 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 6 Jul 2020 14:40:41 +0200 Subject: [PATCH 143/205] Unwrapp API errors to get user-friendly error message Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 60 +++++++--------------------------- ecs/cmd/commands/opts.go | 15 --------- ecs/cmd/commands/secret.go | 24 +++----------- ecs/pkg/amazon/backend/list.go | 9 +++-- ecs/pkg/amazon/backend/logs.go | 15 +++++++-- ecs/pkg/amazon/sdk/sdk.go | 4 ++- ecs/pkg/compose/api.go | 4 +-- ecs/pkg/docker/contextStore.go | 14 ++++++-- 8 files changed, 53 insertions(+), 92 deletions(-) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index ed488a89..9f5a3841 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/compose-spec/compose-go/cli" - "github.com/compose-spec/compose-go/types" "github.com/docker/cli/cli/command" amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" "github.com/docker/ecs-plugin/pkg/docker" @@ -43,15 +42,11 @@ func (o upOptions) LoadBalancerArn() *string { return &o.loadBalancerArn } -func ConvertCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { +func ConvertCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "convert", - RunE: WithProject(projectOpts, func(project *types.Project, args []string) error { - clusteropts, err := docker.GetAwsContext(dockerCli) - if err != nil { - return err - } - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) + RunE: docker.WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { + project, err := cli.ProjectFromOptions(options) if err != nil { return err } @@ -72,36 +67,24 @@ func ConvertCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cob return cmd } -func UpCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { +func UpCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ Use: "up", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) - if err != nil { - return err - } - return backend.Up(context.Background(), *projectOpts) + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + return backend.Up(context.Background(), *options) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") return cmd } -func PsCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { +func PsCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ Use: "ps", - RunE: WithProject(projectOpts, func(project *types.Project, args []string) error { - clusteropts, err := docker.GetAwsContext(dockerCli) - if err != nil { - return err - } - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) - if err != nil { - return err - } - status, err := backend.Ps(context.Background(), project) + RunE: docker.WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { + status, err := backend.Ps(context.Background(), *options) if err != nil { return err } @@ -125,11 +108,7 @@ func DownCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra. opts := downOptions{} cmd := &cobra.Command{ Use: "down", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) - if err != nil { - return err - } + RunE: docker.WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { return backend.Down(context.Background(), *projectOpts) }), } @@ -140,23 +119,8 @@ func DownCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra. func LogsCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "logs [PROJECT NAME]", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) - if err != nil { - return err - } - var name string - - if len(args) == 0 { - project, err := cli.ProjectFromOptions(projectOpts) - if err != nil { - return err - } - name = project.Name - } else { - name = args[0] - } - return backend.Logs(context.Background(), name) + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + return backend.Logs(context.Background(), *projectOpts) }), } return cmd diff --git a/ecs/cmd/commands/opts.go b/ecs/cmd/commands/opts.go index 7d0856c8..bb63ec38 100644 --- a/ecs/cmd/commands/opts.go +++ b/ecs/cmd/commands/opts.go @@ -2,8 +2,6 @@ package commands import ( "github.com/compose-spec/compose-go/cli" - "github.com/compose-spec/compose-go/types" - "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -11,16 +9,3 @@ func AddFlags(o *cli.ProjectOptions, flags *pflag.FlagSet) { flags.StringArrayVarP(&o.ConfigPaths, "file", "f", nil, "Specify an alternate compose file") flags.StringVarP(&o.Name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") } - -type ProjectFunc func(project *types.Project, args []string) error - -// WithProject wrap a ProjectFunc into a cobra command -func WithProject(options *cli.ProjectOptions, f ProjectFunc) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - project, err := cli.ProjectFromOptions(options) - if err != nil { - return err - } - return f(project, args) - } -} diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go index b6a32c78..5bbefaa9 100644 --- a/ecs/cmd/commands/secret.go +++ b/ecs/cmd/commands/secret.go @@ -47,11 +47,7 @@ func CreateSecret(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "create NAME", Short: "Creates a secret.", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) - if err != nil { - return err - } + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { if len(args) == 0 { return errors.New("Missing mandatory parameter: NAME") } @@ -73,11 +69,7 @@ func InspectSecret(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "inspect ID", Short: "Displays secret details", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) - if err != nil { - return err - } + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { if len(args) == 0 { return errors.New("Missing mandatory parameter: ID") } @@ -102,11 +94,7 @@ func ListSecrets(dockerCli command.Cli) *cobra.Command { Use: "list", Aliases: []string{"ls"}, Short: "List secrets stored for the existing account.", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) - if err != nil { - return err - } + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { secrets, err := backend.ListSecrets(context.Background()) if err != nil { return err @@ -125,11 +113,7 @@ func DeleteSecret(dockerCli command.Cli) *cobra.Command { Use: "delete NAME", Aliases: []string{"rm", "remove"}, Short: "Removes a secret.", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, args []string) error { - backend, err := amazon.NewBackend(clusteropts.Profile, clusteropts.Cluster, clusteropts.Region) - if err != nil { - return err - } + RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { if len(args) == 0 { return errors.New("Missing mandatory parameter: [NAME]") } diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index 00b4adee..232b2c81 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -4,11 +4,16 @@ import ( "context" "fmt" - "github.com/compose-spec/compose-go/types" + "github.com/compose-spec/compose-go/cli" "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) Ps(ctx context.Context, project *types.Project) ([]compose.ServiceStatus, error) { +func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose.ServiceStatus, error) { + project, err := cli.ProjectFromOptions(&options) + if err != nil { + return nil, err + } + cluster := b.Cluster if cluster == "" { cluster = project.Name diff --git a/ecs/pkg/amazon/backend/logs.go b/ecs/pkg/amazon/backend/logs.go index a1715885..00b8b2d4 100644 --- a/ecs/pkg/amazon/backend/logs.go +++ b/ecs/pkg/amazon/backend/logs.go @@ -8,11 +8,22 @@ import ( "strconv" "strings" + "github.com/compose-spec/compose-go/cli" + "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) Logs(ctx context.Context, projectName string) error { - err := b.api.GetLogs(ctx, projectName, &logConsumer{ +func (b *Backend) Logs(ctx context.Context, options cli.ProjectOptions) error { + name := options.Name + if name == "" { + project, err := cli.ProjectFromOptions(&options) + if err != nil { + return err + } + name = project.Name + } + + err := b.api.GetLogs(ctx, name, &logConsumer{ colors: map[string]console.ColorFunc{}, width: 0, }) diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 5874a800..3365b07f 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -143,7 +143,9 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) { StackName: aws.String(name), }) if err != nil { - // FIXME doesn't work as expected + if strings.HasPrefix(err.Error(), fmt.Sprintf("ValidationError: Stack with id %s does not exist", name)) { + return false, nil + } return false, nil } return len(stacks.Stacks) > 0, nil diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 64e7e5c8..5f8278c6 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -13,8 +13,8 @@ type API interface { Down(ctx context.Context, options cli.ProjectOptions) error Convert(project *types.Project) (*cloudformation.Template, error) - Logs(ctx context.Context, projectName string) error - Ps(background context.Context, project *types.Project) ([]ServiceStatus, error) + Logs(ctx context.Context, projectName cli.ProjectOptions) error + Ps(background context.Context, options cli.ProjectOptions) ([]ServiceStatus, error) CreateSecret(ctx context.Context, secret Secret) (string, error) InspectSecret(ctx context.Context, id string) (Secret, error) diff --git a/ecs/pkg/docker/contextStore.go b/ecs/pkg/docker/contextStore.go index b038eb92..83c055f6 100644 --- a/ecs/pkg/docker/contextStore.go +++ b/ecs/pkg/docker/contextStore.go @@ -3,9 +3,11 @@ package docker import ( "fmt" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/docker/cli/cli/command" cliconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/context/store" + amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" "github.com/mitchellh/mapstructure" "github.com/spf13/cobra" ) @@ -72,7 +74,7 @@ func checkAwsContextExists(contextName string) (*AwsContext, error) { return &awsContext, nil } -type ContextFunc func(ctx AwsContext, args []string) error +type ContextFunc func(ctx AwsContext, backend *amazon.Backend, args []string) error func WithAwsContext(dockerCli command.Cli, f ContextFunc) func(cmd *cobra.Command, args []string) error { return func(cmd *cobra.Command, args []string) error { @@ -80,7 +82,15 @@ func WithAwsContext(dockerCli command.Cli, f ContextFunc) func(cmd *cobra.Comman if err != nil { return err } - return f(*ctx, args) + backend, err := amazon.NewBackend(ctx.Profile, ctx.Cluster, ctx.Region) + if err != nil { + return err + } + err = f(*ctx, backend, args) + if e, ok := err.(awserr.Error); ok { + return fmt.Errorf(e.Message()) + } + return err } } From 242216cab1ffc2ea4b14bf32550f5936e8ef8b1b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 7 Jul 2020 11:50:42 +0200 Subject: [PATCH 144/205] Reject compose file not setting service image Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 1 + ecs/pkg/amazon/backend/convert.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 3df5b8ab..5a096237 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -40,6 +40,7 @@ func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortCo } if p.Published != p.Target { c.Error("published port can't be set to a distinct value than container port") + p.Published = p.Target } } diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index 0eace3a6..196ff9ac 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -17,6 +17,10 @@ import ( ) func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { + if service.Image == "" { + return nil, fmt.Errorf("service %s doesn't define a Docker image to run", service.Name) + } + cpu, mem, err := toLimits(service) if err != nil { return nil, err From 98ec6c173b5e53d8109dabaabb68e649c9f479fb Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 7 Jul 2020 14:50:52 +0200 Subject: [PATCH 145/205] Reject compose file that uses incompatible features Signed-off-by: Nicolas De Loof --- ecs/go.mod | 2 +- ecs/go.sum | 4 ++++ ecs/pkg/amazon/backend/cloudformation.go | 15 +++++++++++---- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/ecs/go.mod b/ecs/go.mod index 163b56fc..cdfa6c31 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -14,7 +14,7 @@ require ( github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 + github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9 github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 diff --git a/ecs/go.sum b/ecs/go.sum index 27588bfe..f03fc147 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -56,6 +56,8 @@ github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:ea github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 h1:sVvKsoXizFOuJNc8dM91IeET2/zDNFj3hwHgk437iJ8= github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= +github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9 h1:WkFqc6UpRqxROso9KC+ceaTiXx/VWpeO1x+NV0d4d+o= +github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -167,6 +169,8 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 5a096237..4e0f3099 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -17,6 +17,7 @@ import ( cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/compose-spec/compose-go/compatibility" + "github.com/compose-spec/compose-go/errdefs" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" @@ -39,8 +40,7 @@ func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortCo p.Published = p.Target } if p.Published != p.Target { - c.Error("published port can't be set to a distinct value than container port") - p.Published = p.Target + c.Incompatible("published port can't be set to a distinct value than container port") } } @@ -51,7 +51,7 @@ func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) case "SYS_PTRACE": add = append(add, cap) default: - c.Error("service.cap_add = %s", cap) + c.Incompatible("ECS doesn't allow to add capability %s", cap) } } service.CapAdd = add @@ -86,7 +86,14 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro } compatibility.Check(project, checker) for _, err := range checker.Errors() { - logrus.Warn(err.Error()) + if errdefs.IsIncompatibleError(err) { + logrus.Error(err.Error()) + } else { + logrus.Warn(err.Error()) + } + } + if !compatibility.IsCompatible(checker) { + return nil, fmt.Errorf("compose file is incompatible with Amazon ECS") } template := cloudformation.NewTemplate() From 6664447d291eb09eb6812203c15847bd3bf9bca9 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 8 Jul 2020 12:00:25 +0200 Subject: [PATCH 146/205] Fix setup command breaks if .aws/config does not exists Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/setup.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index 6d52664f..be876d66 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -3,6 +3,7 @@ package commands import ( "fmt" "os" + "path/filepath" "reflect" "strings" @@ -101,10 +102,17 @@ func saveCredentials(profile string, accessKeyID string, secretAccessKey string) } if err.(awserr.Error).Code() == "SharedCredsLoad" && err.(awserr.Error).Message() == "failed to load shared credentials file" { - os.Create(p.Filename) + err = os.MkdirAll(filepath.Dir(p.Filename), 0700) + if err != nil { + return err + } + _, err = os.Create(p.Filename) + if err != nil { + return err + } } - credIni, err := ini.Load(p.Filename) + credIni, err := ini.LooseLoad(p.Filename) if err != nil { return err } @@ -122,7 +130,7 @@ func awsProfiles(filename string) (map[string]ini.Section, error) { if filename == "" { filename = defaults.SharedConfigFilename() } - credIni, err := ini.Load(filename) + credIni, err := ini.LooseLoad(filename) if err != nil { return nil, err } From ec58975524ff95e69800eba50cf4e68e228205e3 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 8 Jul 2020 16:07:13 +0200 Subject: [PATCH 147/205] Don't prepent docker.io to image URI. Let the container runtime apply default registry Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation_test.go | 2 +- ecs/pkg/amazon/backend/convert.go | 13 +------------ .../simple/simple-cloudformation-conversion.golden | 2 +- ...-cloudformation-with-overrides-conversion.golden | 2 +- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 2fcd1542..518c7c77 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -128,7 +128,7 @@ services: `) def := template.Resources["TestTaskDefinition"].(*ecs.TaskDefinition) container := def.ContainerDefinitions[0] - assert.Equal(t, container.Image, "docker.io/library/image") + assert.Equal(t, container.Image, "image") assert.Equal(t, container.Command[0], "command") assert.Equal(t, container.EntryPoint[0], "entrypoint") assert.Equal(t, get(container.Environment, "FOO"), "BAR") diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index 196ff9ac..ee5aa95b 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -52,7 +52,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi FirelensConfiguration: nil, HealthCheck: toHealthCheck(service.HealthCheck), Hostname: service.Hostname, - Image: getImage(service.Image), + Image: service.Image, Interactive: false, Links: nil, LinuxParameters: toLinuxParameters(service), @@ -308,17 +308,6 @@ func toKeyValuePair(environment types.MappingWithEquals) []ecs.TaskDefinition_Ke return pairs } -func getImage(image string) string { - switch f := strings.Split(image, "/"); len(f) { - case 1: - return "docker.io/library/" + image - case 2: - return "docker.io/" + image - default: - return image - } -} - func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name for key, value := range service.Extensions { diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden index b9fd6a04..0256a586 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden @@ -174,7 +174,7 @@ } ], "Essential": true, - "Image": "docker.io/library/nginx", + "Image": "nginx", "LinuxParameters": {}, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index 0f93cbd5..b2d1ffad 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -174,7 +174,7 @@ } ], "Essential": true, - "Image": "docker.io/library/haproxy", + "Image": "haproxy", "LinuxParameters": {}, "LogConfiguration": { "LogDriver": "awslogs", From 6f916ab9cedd35d2cf59f71a0ac6e45b63ae8346 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 7 Jul 2020 10:43:15 +0200 Subject: [PATCH 148/205] Update docs with download and docs links Signed-off-by: Nicolas De Loof --- ecs/docs/get-started-linux.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ecs/docs/get-started-linux.md b/ecs/docs/get-started-linux.md index 2b1a129b..1b24c84f 100644 --- a/ecs/docs/get-started-linux.md +++ b/ecs/docs/get-started-linux.md @@ -19,15 +19,14 @@ Linux it needs to be installed manually. You can download the Docker ECS plugin from this repository using the following command: - ```console -$ curl -L http://xxx | tar xzf - +$ curl -L https://github.com/docker/ecs-plugin/releases/latest/download/docker-ecs-linux-amd64 ``` You will then need to make it executable: ```console -$ chmod +x docker-ecs +$ chmod +x docker-ecs-linux-amd64 ``` ### Plugin install @@ -38,7 +37,7 @@ it to the right place: ```console $ mkdir -p /usr/local/lib/docker/cli-plugins -$ mv docker-ecs /usr/local/lib/docker/cli-plugins/ +$ mv docker-ecs-linux-amd64 /usr/local/lib/docker/cli-plugins/docker-ecs ``` You can put the CLI plugin into any of the following directories: @@ -80,5 +79,4 @@ $ docker ecs version Docker ECS plugin 0.0.1 ``` - -You are now ready to [start deploying to ECS](http://xxx) +You are now ready to [start deploying to ECS](https://docs.docker.com/engine/context/ecs-integration/) From 2586fa35d42a94becd6660232b0cc8cd81721ffc Mon Sep 17 00:00:00 2001 From: Chad Metcalf Date: Tue, 7 Jul 2020 22:20:27 -0700 Subject: [PATCH 149/205] Adding the demo from AWS C3. Signed-off-by: Nicolas De Loof --- ecs/demo/Makefile | 25 ++++++ ecs/demo/README.md | 107 ++++++++++++++++++++++++ ecs/demo/app/Dockerfile | 7 ++ ecs/demo/app/app.py | 19 +++++ ecs/demo/app/requirements.txt | 2 + ecs/demo/app/scripts/entrypoint.sh | 4 + ecs/demo/app/templates/index.html | 125 +++++++++++++++++++++++++++++ ecs/demo/docker-compose.yml | 12 +++ 8 files changed, 301 insertions(+) create mode 100644 ecs/demo/Makefile create mode 100644 ecs/demo/README.md create mode 100644 ecs/demo/app/Dockerfile create mode 100644 ecs/demo/app/app.py create mode 100644 ecs/demo/app/requirements.txt create mode 100755 ecs/demo/app/scripts/entrypoint.sh create mode 100644 ecs/demo/app/templates/index.html create mode 100644 ecs/demo/docker-compose.yml diff --git a/ecs/demo/Makefile b/ecs/demo/Makefile new file mode 100644 index 00000000..deba2b83 --- /dev/null +++ b/ecs/demo/Makefile @@ -0,0 +1,25 @@ +REPO_NAMESPACE ?= ${USER} +FRONTEND_IMG = ${REPO_NAMESPACE}/timestamper +REGISTRY_ID ?= PUT_ECR_REGISTRY_ID_HERE +DOCKER_PUSH_REPOSITORY=dkr.ecr.us-west-2.amazonaws.com + +all: build-image + +create-ecr: + aws ecr create-repository --repository-name ${FRONTEND_IMG} + +build-image: + docker build -t $(REGISTRY_ID).$(DOCKER_PUSH_REPOSITORY)/$(FRONTEND_IMG) ./app + docker build -t $(FRONTEND_IMG) ./app + +push-image-ecr: + aws ecr get-login-password --region us-west-2 | docker login -u AWS --password-stdin $(REGISTRY_ID).$(DOCKER_PUSH_REPOSITORY) + docker push $(REGISTRY_ID).$(DOCKER_PUSH_REPOSITORY)/$(FRONTEND_IMG) + +push-image-hub: + docker push $(FRONTEND_IMG) + +clean: + @docker context use default + @docker context rm aws || true + @docker-compose rm -f || true diff --git a/ecs/demo/README.md b/ecs/demo/README.md new file mode 100644 index 00000000..80323520 --- /dev/null +++ b/ecs/demo/README.md @@ -0,0 +1,107 @@ +## Compose sample application + +### Python/Flask application + +``` ++--------------------+ +------------------+ +| | | | +| Python Flask | timestamps | Redis | +| Application |------------->| | +| | | | ++--------------------+ +------------------+ +``` + +### Things you'll need to do. + +There are a number of places you'll need to fill in information. You can find them with: + +``` +grep -r '<<<' ./* +./docker-compose.yml: x-aws-pull_credentials: <<>> +./docker-compose.yml: image: <<>>/timestamper +## Walk through +``` + +### Setup pull credentials for private Docker Hub repositories + +You should use a Personal Access Token (PAT) vs your default password. If you have 2FA enabled on your Hub account you will have to create a PAT. You can read more about managing access tokens here: https://docs.docker.com/docker-hub/access-tokens/ + +``` + docker ecs secret create -d MyKey -u myhubusername -p myhubpat +``` + +### Create an AWS Docker context and list available contexts + +``` +docker ecs setup +Enter context name: aws +✔ sandbox.devtools.developer +Enter cluster name: +Enter region: us-west-2 +✗ Enter credentials: + +docker context ls +NAME DESCRIPTION DOCKER ENDPOINT KUBERNETES ENDPOINT ORCHESTRATOR +aws +default * Current DOCKER_HOST based configuration unix:///var/run/docker.sock swarm +``` + +### Test locally + +``` +docker context use default +docker-compose up +open http://localhost:5000 +``` + +### Push images to hub for ecs (ecs cannot see your local image cache) + +``` +docker-compose push +``` + +### Switch to ECS context and launch the app + +``` +docker context use aws +docker ecs compose up +``` + +### Check out the CLI + +``` +docker ecs compose ps +docker ecs compose logs +``` + +### Check out the aws console + +- cloud formation +- cloud watch +- security groups +- Load balancers (ELB for this example / ALB if your app only uses 80/443) + +### Checkout cloudformation + +``` +docker ecs compose convert +``` + +### Stop the meters + +``` +docker ecs compose down + +``` + +## Using Amazon ECR instead of Docker Hub + +[Makefile](Makefile) has an example setup for creating an ECR repository and pushing to it. You'll need to have the AWS CLI installed and your AWS credentials available. + +``` +make create-ecr +REGISTRY_ID= make build-image +REGISTRY_ID= make push-image-ecr +``` + +If you want to use this often, you'll likely want to replace `PUT_ECR_REGISTRY_ID_HERE` with the value from above. diff --git a/ecs/demo/app/Dockerfile b/ecs/demo/app/Dockerfile new file mode 100644 index 00000000..64469d28 --- /dev/null +++ b/ecs/demo/app/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.7-alpine +WORKDIR /app +COPY requirements.txt /app +RUN pip3 install -r requirements.txt +COPY . /app +ENTRYPOINT ["/app/scripts/entrypoint.sh"] +CMD ["python3", "app.py"] diff --git a/ecs/demo/app/app.py b/ecs/demo/app/app.py new file mode 100644 index 00000000..e54b19a2 --- /dev/null +++ b/ecs/demo/app/app.py @@ -0,0 +1,19 @@ + +from flask import Flask +from flask import render_template +from redis import StrictRedis +from datetime import datetime + +app = Flask(__name__) +redis = StrictRedis(host='backend', port=6379) + + +@app.route('/') +def home(): + redis.lpush('times', datetime.now().strftime('%Y-%m-%dT%H:%M:%S%z')) + return render_template('index.html', title='Home', + times=redis.lrange('times', 0, -1)) + + +if __name__ == '__main__': + app.run(host='0.0.0.0', debug=True) diff --git a/ecs/demo/app/requirements.txt b/ecs/demo/app/requirements.txt new file mode 100644 index 00000000..1a5dc97b --- /dev/null +++ b/ecs/demo/app/requirements.txt @@ -0,0 +1,2 @@ +flask +redis diff --git a/ecs/demo/app/scripts/entrypoint.sh b/ecs/demo/app/scripts/entrypoint.sh new file mode 100755 index 00000000..19866024 --- /dev/null +++ b/ecs/demo/app/scripts/entrypoint.sh @@ -0,0 +1,4 @@ +#! /bin/sh + +if [ "${LOCALDOMAIN}" != "" ]; then echo "search ${LOCALDOMAIN}" >> /etc/resolv.conf; fi +exec "$@" diff --git a/ecs/demo/app/templates/index.html b/ecs/demo/app/templates/index.html new file mode 100644 index 00000000..91efdaef --- /dev/null +++ b/ecs/demo/app/templates/index.html @@ -0,0 +1,125 @@ + + + + + + + + + + + Hello, Docker! + + +
+ + +
+

Hello, Docker Folks!

+
+ +
+
+ + + Timestamp! + +
+ +
+ +
+ + + + + + + + {% for t in times %} + + + + {% endfor %} + +
Timestamp
{{ t.decode('utf-8') }}
+
+ +
+ + + + + + + + diff --git a/ecs/demo/docker-compose.yml b/ecs/demo/docker-compose.yml new file mode 100644 index 00000000..31152b2a --- /dev/null +++ b/ecs/demo/docker-compose.yml @@ -0,0 +1,12 @@ +version: "3.8" +services: + frontend: + build: app + x-aws-pull_credentials: <<>> + image: <<>>/timestamper + ports: + - "5000:5000" + depends_on: + - backend + backend: + image: redis:alpine From 52a64845c7714cc68c233ac19c74b92498b0bf9f Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 9 Jul 2020 11:51:07 +0200 Subject: [PATCH 150/205] example: Add details and format Signed-off-by: Christopher Crone Signed-off-by: Nicolas De Loof --- ecs/demo/README.md | 107 ----------- ecs/{demo => example}/Makefile | 0 ecs/example/README.md | 178 ++++++++++++++++++ ecs/{demo => example}/app/Dockerfile | 0 ecs/{demo => example}/app/app.py | 0 ecs/{demo => example}/app/requirements.txt | 0 .../app/scripts/entrypoint.sh | 0 .../app/templates/index.html | 0 ecs/{demo => example}/docker-compose.yml | 2 +- 9 files changed, 179 insertions(+), 108 deletions(-) delete mode 100644 ecs/demo/README.md rename ecs/{demo => example}/Makefile (100%) create mode 100644 ecs/example/README.md rename ecs/{demo => example}/app/Dockerfile (100%) rename ecs/{demo => example}/app/app.py (100%) rename ecs/{demo => example}/app/requirements.txt (100%) rename ecs/{demo => example}/app/scripts/entrypoint.sh (100%) rename ecs/{demo => example}/app/templates/index.html (100%) rename ecs/{demo => example}/docker-compose.yml (81%) diff --git a/ecs/demo/README.md b/ecs/demo/README.md deleted file mode 100644 index 80323520..00000000 --- a/ecs/demo/README.md +++ /dev/null @@ -1,107 +0,0 @@ -## Compose sample application - -### Python/Flask application - -``` -+--------------------+ +------------------+ -| | | | -| Python Flask | timestamps | Redis | -| Application |------------->| | -| | | | -+--------------------+ +------------------+ -``` - -### Things you'll need to do. - -There are a number of places you'll need to fill in information. You can find them with: - -``` -grep -r '<<<' ./* -./docker-compose.yml: x-aws-pull_credentials: <<>> -./docker-compose.yml: image: <<>>/timestamper -## Walk through -``` - -### Setup pull credentials for private Docker Hub repositories - -You should use a Personal Access Token (PAT) vs your default password. If you have 2FA enabled on your Hub account you will have to create a PAT. You can read more about managing access tokens here: https://docs.docker.com/docker-hub/access-tokens/ - -``` - docker ecs secret create -d MyKey -u myhubusername -p myhubpat -``` - -### Create an AWS Docker context and list available contexts - -``` -docker ecs setup -Enter context name: aws -✔ sandbox.devtools.developer -Enter cluster name: -Enter region: us-west-2 -✗ Enter credentials: - -docker context ls -NAME DESCRIPTION DOCKER ENDPOINT KUBERNETES ENDPOINT ORCHESTRATOR -aws -default * Current DOCKER_HOST based configuration unix:///var/run/docker.sock swarm -``` - -### Test locally - -``` -docker context use default -docker-compose up -open http://localhost:5000 -``` - -### Push images to hub for ecs (ecs cannot see your local image cache) - -``` -docker-compose push -``` - -### Switch to ECS context and launch the app - -``` -docker context use aws -docker ecs compose up -``` - -### Check out the CLI - -``` -docker ecs compose ps -docker ecs compose logs -``` - -### Check out the aws console - -- cloud formation -- cloud watch -- security groups -- Load balancers (ELB for this example / ALB if your app only uses 80/443) - -### Checkout cloudformation - -``` -docker ecs compose convert -``` - -### Stop the meters - -``` -docker ecs compose down - -``` - -## Using Amazon ECR instead of Docker Hub - -[Makefile](Makefile) has an example setup for creating an ECR repository and pushing to it. You'll need to have the AWS CLI installed and your AWS credentials available. - -``` -make create-ecr -REGISTRY_ID= make build-image -REGISTRY_ID= make push-image-ecr -``` - -If you want to use this often, you'll likely want to replace `PUT_ECR_REGISTRY_ID_HERE` with the value from above. diff --git a/ecs/demo/Makefile b/ecs/example/Makefile similarity index 100% rename from ecs/demo/Makefile rename to ecs/example/Makefile diff --git a/ecs/example/README.md b/ecs/example/README.md new file mode 100644 index 00000000..96e3e526 --- /dev/null +++ b/ecs/example/README.md @@ -0,0 +1,178 @@ +## Compose sample application + +This sample application was demoed as part of the AWS Cloud Containers +Conference on 2020-07-09. It has been tested on Linux and macOS. + +Note that `$` is used to denote commands in blocks where the command and its +output are included. + +### Python/Flask application + +``` ++--------------------+ +------------------+ +| | | | +| Python Flask | timestamps | Redis | +| Application |------------->| | +| | | | ++--------------------+ +------------------+ +``` + +### Things you'll need to do + +There are a number of places you'll need to fill in information. +You can find them with: + +```console +$ grep -r '<<<' ./* +./docker-compose.yml: x-aws-pull_credentials: <<>> +./docker-compose.yml: image: <<>>/timestamper +## Walk through +``` + +### Setup pull credentials for private Docker Hub repositories + +You should use a Personal Access Token (PAT) rather than your account password. +If you have 2FA enabled on your Hub account you will need to create a PAT. +You can read more about managing access tokens here: +https://docs.docker.com/docker-hub/access-tokens/ + +```console +docker ecs secret create -d MyKey -u myhubusername -p myhubpat +``` + +### Create an AWS Docker context and list available contexts + +To initialize the Docker ECS integration, you will need to run the `setup` +command. This will create a Docker context that works with AWS ECS. + +```console +$ docker ecs setup +Enter context name: aws +✔ sandbox.devtools.developer +Enter cluster name: +Enter region: us-west-2 +✗ Enter credentials: +``` + +You can verify that the context was created by listing your Docker contexts: + +```console +$ docker context ls +NAME DESCRIPTION DOCKER ENDPOINT KUBERNETES ENDPOINT ORCHESTRATOR +aws +default * Current DOCKER_HOST based configuration unix:///var/run/docker.sock swarm +``` + +### Test locally + +The first step is to test your application works locally. To do this, you will +need to switch to using the default local context so that you are targeting your +local machine. + +```console +docker context use default +``` + +You can then run the application using `docker-compose`: + +```console +docker-compose up +``` + +Once the application has started, you can navigate to http://localhost:5000 +using your Web browser using the following command: + +```console +open http://localhost:5000 +``` + +### Push images to Docker Hub for ECS (ECS cannot see your local image cache) + +In order to run your application in the cloud, you will need your container +images to be in a registry. You can push them from your local machine using: + +```console +docker-compose push +``` + +You can verify that this command pushed to the Docker Hub by +[logging in](https://hub.docker.com) and looking for the `timestamper` +repository under your user name. + +### Switch to ECS context and launch the app + +Now that you've tested the application works locally and that you've pushed the +container images to the Docker Hub, you can switch to using the `aws` context +you created earlier. + +```console +docker context use aws +``` + +Running the application on ECS is then as simple as doing a `compose up`: + +```console +docker ecs compose up +``` + +### Check out the CLI + +Once the application is running in ECS, you can list the running containers with +the `ps` command. Note that you will need to run this from the directory where +you Compose file is. + +```console +docker ecs compose ps +``` + +You can also read the application logs using `compose logs`: + +```console +docker ecs compose logs +``` + +### Check out the AWS console + +You can see all the AWS components created for your running application in the +AWS console. There you will find: + +- CloudFormation being used to manage all the infrastructure +- CloudWatch for logs +- Security Groups for network policies +- Load balancers (ELB for this example / ALB if your app only uses 80/443) + +### Checkout CloudFormation + +The ECS Docker CLI integration has the ability to output the CloudFormation +template used to create the application in the `compose convert` command. You +can see this by running: + +```console +docker ecs compose convert +``` + +### Stop the meters + +To shut down your application, you simply need to run: + +```console +docker ecs compose down +``` + +## Using Amazon ECR instead of Docker Hub + +If you'd like to use AWS ECR instead of Docker Hub, the [Makefile](Makefile) has +an example setup for creating an ECR repository and pushing to it. +You'll need to have the AWS CLI installed and your AWS credentials available. + +```console +make create-ecr +REGISTRY_ID= make build-image +REGISTRY_ID= make push-image-ecr +``` + +Note that you will need to change the name of the image in the +[Compose file](docker-compose.yml). + +If you want to use this often, you'll likely want to replace +`PUT_ECR_REGISTRY_ID_HERE` with the value from above. diff --git a/ecs/demo/app/Dockerfile b/ecs/example/app/Dockerfile similarity index 100% rename from ecs/demo/app/Dockerfile rename to ecs/example/app/Dockerfile diff --git a/ecs/demo/app/app.py b/ecs/example/app/app.py similarity index 100% rename from ecs/demo/app/app.py rename to ecs/example/app/app.py diff --git a/ecs/demo/app/requirements.txt b/ecs/example/app/requirements.txt similarity index 100% rename from ecs/demo/app/requirements.txt rename to ecs/example/app/requirements.txt diff --git a/ecs/demo/app/scripts/entrypoint.sh b/ecs/example/app/scripts/entrypoint.sh similarity index 100% rename from ecs/demo/app/scripts/entrypoint.sh rename to ecs/example/app/scripts/entrypoint.sh diff --git a/ecs/demo/app/templates/index.html b/ecs/example/app/templates/index.html similarity index 100% rename from ecs/demo/app/templates/index.html rename to ecs/example/app/templates/index.html diff --git a/ecs/demo/docker-compose.yml b/ecs/example/docker-compose.yml similarity index 81% rename from ecs/demo/docker-compose.yml rename to ecs/example/docker-compose.yml index 31152b2a..9d08d45d 100644 --- a/ecs/demo/docker-compose.yml +++ b/ecs/example/docker-compose.yml @@ -3,7 +3,7 @@ services: frontend: build: app x-aws-pull_credentials: <<>> - image: <<>>/timestamper + image: <<>>/timestamper ports: - "5000:5000" depends_on: From 164e0d750d9098f5c0eeadcc2677444b937f7f04 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 9 Jul 2020 11:54:03 +0200 Subject: [PATCH 151/205] readme: Add link to example Signed-off-by: Christopher Crone Signed-off-by: Nicolas De Loof --- ecs/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ecs/README.md b/ecs/README.md index a4e0caf5..bf721a3a 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -4,6 +4,10 @@ :exclamation: The Docker ECS plugin is still in Beta. It's design and UX will evolve until 1.0 Final release. +## Example + +You can find an application for testing this in [example](./example). + ## Architecture ECS plugin is a [Docker CLI plugin](https://docs.docker.com/engine/extend/cli_plugins/) From bdb8ad0b95234c5f8827286355d824578eb09f71 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 9 Jul 2020 12:25:34 +0200 Subject: [PATCH 152/205] readme: Tidy and add docs link Signed-off-by: Christopher Crone Signed-off-by: Nicolas De Loof --- ecs/README.md | 86 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/ecs/README.md b/ecs/README.md index bf721a3a..d948263b 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -2,21 +2,34 @@ ## Status -:exclamation: The Docker ECS plugin is still in Beta. It's design and UX will evolve until 1.0 Final release. +:exclamation: The Docker ECS plugin is still in Beta. +Its design and UX will evolve until 1.0 Final release. -## Example +## Example and documentation You can find an application for testing this in [example](./example). +You can find more documentation about using the Docker ECS integration +[here](https://docs.docker.com/engine/context/ecs-integration/). + ## Architecture -ECS plugin is a [Docker CLI plugin](https://docs.docker.com/engine/extend/cli_plugins/) -root command `ecs` require aws profile to get API credentials from `~/.aws/credentials` -as well as AWS region - those will later be stored in a docker context +The Docker ECS integration is a +[Docker CLI plugin](https://docs.docker.com/engine/extend/cli_plugins/) +with the root command of `ecs`. +It requires an AWS profile to select the AWS API credentials from +`~/.aws/credentials` as well as an AWS region. You can read more about CLI AWS +credential management +[here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +Once setup, the AWS profile and region are stored in a Docker context. -A `compose.yaml` is parsed and converted into a [CloudFormation](https://aws.amazon.com/cloudformation/) -template, which will create all resources in dependent order and cleanup on -`down` command or deployment failure. +A `compose.yaml` file is parsed and converted into a +[CloudFormation](https://aws.amazon.com/cloudformation/) template, +which is then used to create all application resources in dependent order. +Resources are cleaned up with the `down` command or in the case of a deployment +failure. + +The architecture of the ECS integration is shown below: ``` +--------------------------------------+ @@ -24,48 +37,61 @@ template, which will create all resources in dependent order and cleanup on +--------------------------------------+ - Load +--------------------------------------+ - | compose Model | + | Compose Model | +--------------------------------------+ - Validate +--------------------------------------+ - | compose Model suitable for ECS | + | Compose Model suitable for ECS | +--------------------------------------+ - Convert +--------------------------------------+ | CloudFormation Template | +--------------------------------------+ - Apply - +--------------+ +----------------+ + +--------------+ +----------------+ | AWS API | or | stack file | +--------------+ +----------------+ ``` -* _Load_ phase relies on [compose-go](https://github.com/compose-spec/compose-go). Any generic code we write for this -purpose should be proposed upstream. -* _Validate_ phase is responsible to inject sane ECS defaults into the compose-go model, and validate the `compose.yaml` -file do not include unsupported features. -* _Convert_ produces a CloudFormation template to define all resources required to implement the application model on AWS. -* _Apply_ phase do apply the CloudFormation template, either by exporting to a stack file or to deploy on AWS. +* The _Load_ phase relies on + [compose-go](https://github.com/compose-spec/compose-go). + Any generic code we write for this purpose should be proposed upstream. +* The _Validate_ phase is responsible for injecting sane ECS defaults into the + compose-go model, and validating the `compose.yaml` file does not include + unsupported features. +* The _Convert_ phase produces a CloudFormation template to define all + application resources required to implement the application model on AWS. +* The _Apply_ phase does the actual apply of the CloudFormation template, + either by exporting to a stack file or to deploy on AWS. ## Application model ### Services -Compose services are mapped to ECS services. Compose specification has no support for multi-container services, nor -does it support sidecars. When an ECS feature requires a sidecar, we introduce custom Compose extension (`x-aws-*`) -to actually expose ECS feature as a service-level feature, not plumbing details. +Compose services are mapped to ECS services. The Compose specification does not +have support for multi-container services (like Kubernetes pods) or sidecars. +When an ECS feature requires a sidecar, we use a custom Compose extension +(`x-aws-*`) to expose the ECS features as a service-level feature, +and keep the plumbing details from the user. ### Networking -We map the "network" abstraction from Compose model to AWS SecurityGroups. The whole application is created within a -single VPC, SecurityGroups are created per networks, including the implicit `default` one. Services are attached -according to the networks declared in Compose model. Doing so, services attached to a common security group can -communicate together, while services from distinct SecurityGroups can't. We just can't set service aliasses per network. +We map the "network" abstraction from the Compose model to AWS security groups. +The whole application is created within a single VPC, +security groups are created per Compose network, including the implicit +`default` one. +Services are attached according to the networks declared in Compose model. +This approach means that services attached to a common security group can +communicate together, while services from distinct security groups cannot. +This matches the intent of the Compose network model with the limitation that we +cannot set service aliases per network. + +A [CloudMap](https://aws.amazon.com/cloud-map/) private namespace is created for +each application as `{project}.local`. Services get registered so that we +have service discovery and DNS round-robin +(equivalent to Compose's `endpoint_mode: dnsrr`). Docker images SHOULD include +an entrypoint script to replicate this feature: -A CloudMap private namespace is created for application as `{project}.local`. Services get registered so that we -get service discovery and DNS round-robin (equivalent for Compose's `endpoint_mode: dnsrr`). Docker images SHOULD -include a tiny entrypoint script to replicate this feature: ```shell script -if [ ! -z LOCALDOMAIN ]; then echo "search ${LOCALDOMAIN}" >> /etc/resolv.conf; fi -``` - +if [ ! -z LOCALDOMAIN ]; then echo "search ${LOCALDOMAIN}" >> /etc/resolv.conf; fi +``` From 16e7bbd69726b9971afafaa21f65b7f350307dc4 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 9 Jul 2020 10:49:21 +0200 Subject: [PATCH 153/205] Use compatibilityChecker to detect missing service image as a blocker Signed-off-by: Nicolas De Loof --- ecs/go.mod | 2 +- ecs/go.sum | 2 + ecs/pkg/amazon/backend/cloudformation.go | 48 +------------------ ecs/pkg/amazon/backend/compatibility.go | 61 ++++++++++++++++++++++++ ecs/pkg/amazon/backend/convert.go | 4 -- 5 files changed, 65 insertions(+), 52 deletions(-) create mode 100644 ecs/pkg/amazon/backend/compatibility.go diff --git a/ecs/go.mod b/ecs/go.mod index cdfa6c31..332a4688 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -14,7 +14,7 @@ require ( github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9 + github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 diff --git a/ecs/go.sum b/ecs/go.sum index f03fc147..9fa33dd9 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -58,6 +58,8 @@ github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 h1:sVvKsoX github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9 h1:WkFqc6UpRqxROso9KC+ceaTiXx/VWpeO1x+NV0d4d+o= github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= +github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a h1:pIiSz5jML7rQ1aupg/KHlTqCxhyXvIgeDMf4kDTzIg8= +github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 4e0f3099..bf8ecb3f 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -31,57 +31,11 @@ const ( ParameterLoadBalancerARN = "ParameterLoadBalancerARN" ) -type FargateCompatibilityChecker struct { - compatibility.AllowList -} - -func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortConfig) { - if p.Published == 0 { - p.Published = p.Target - } - if p.Published != p.Target { - c.Incompatible("published port can't be set to a distinct value than container port") - } -} - -func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) { - add := []string{} - for _, cap := range service.CapAdd { - switch cap { - case "SYS_PTRACE": - add = append(add, cap) - default: - c.Incompatible("ECS doesn't allow to add capability %s", cap) - } - } - service.CapAdd = add -} - // Convert a compose project into a CloudFormation template func (b Backend) Convert(project *types.Project) (*cloudformation.Template, error) { var checker compatibility.Checker = &FargateCompatibilityChecker{ compatibility.AllowList{ - Supported: []string{ - "services.command", - "services.container_name", - "services.cap_drop", - "services.depends_on", - "services.entrypoint", - "services.environment", - "services.init", - "services.healthcheck", - "services.healthcheck.interval", - "services.healthcheck.start_period", - "services.healthcheck.test", - "services.healthcheck.timeout", - "services.networks", - "services.ports", - "services.ports.mode", - "services.ports.target", - "services.ports.protocol", - "services.user", - "services.working_dir", - }, + Supported: compatibleComposeAttributes, }, } compatibility.Check(project, checker) diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/pkg/amazon/backend/compatibility.go new file mode 100644 index 00000000..bc74bd87 --- /dev/null +++ b/ecs/pkg/amazon/backend/compatibility.go @@ -0,0 +1,61 @@ +package backend + +import ( + "github.com/compose-spec/compose-go/compatibility" + "github.com/compose-spec/compose-go/types" +) + +type FargateCompatibilityChecker struct { + compatibility.AllowList +} + +var compatibleComposeAttributes = []string{ + "services.command", + "services.container_name", + "services.cap_drop", + "services.depends_on", + "services.entrypoint", + "services.environment", + "service.image", + "services.init", + "services.healthcheck", + "services.healthcheck.interval", + "services.healthcheck.start_period", + "services.healthcheck.test", + "services.healthcheck.timeout", + "services.networks", + "services.ports", + "services.ports.mode", + "services.ports.target", + "services.ports.protocol", + "services.user", + "services.working_dir", +} + +func (c *FargateCompatibilityChecker) CheckImage(service *types.ServiceConfig) { + if service.Image == "" { + c.Incompatible("service %s doesn't define a Docker image to run", service.Name) + } +} + +func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortConfig) { + if p.Published == 0 { + p.Published = p.Target + } + if p.Published != p.Target { + c.Incompatible("published port can't be set to a distinct value than container port") + } +} + +func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) { + add := []string{} + for _, cap := range service.CapAdd { + switch cap { + case "SYS_PTRACE": + add = append(add, cap) + default: + c.Incompatible("ECS doesn't allow to add capability %s", cap) + } + } + service.CapAdd = add +} diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index ee5aa95b..6fdc0abc 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -17,10 +17,6 @@ import ( ) func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { - if service.Image == "" { - return nil, fmt.Errorf("service %s doesn't define a Docker image to run", service.Name) - } - cpu, mem, err := toLimits(service) if err != nil { return nil, err From bcd9cda41c66def61f907c98ae05ec30a7f78c10 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 9 Jul 2020 18:01:12 +0200 Subject: [PATCH 154/205] docs: Fix download command Signed-off-by: Christopher Crone Signed-off-by: Nicolas De Loof --- ecs/docs/get-started-linux.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/docs/get-started-linux.md b/ecs/docs/get-started-linux.md index 1b24c84f..afada890 100644 --- a/ecs/docs/get-started-linux.md +++ b/ecs/docs/get-started-linux.md @@ -20,7 +20,7 @@ You can download the Docker ECS plugin from this repository using the following command: ```console -$ curl -L https://github.com/docker/ecs-plugin/releases/latest/download/docker-ecs-linux-amd64 +$ curl -LO https://github.com/docker/ecs-plugin/releases/latest/download/docker-ecs-linux-amd64 ``` You will then need to make it executable: From 0b5779b52d50a5c47d9802668fa4f84b64e72d52 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 9 Jul 2020 18:09:47 +0200 Subject: [PATCH 155/205] readme: Add link to blog, install instructions Signed-off-by: Christopher Crone Signed-off-by: Nicolas De Loof --- ecs/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ecs/README.md b/ecs/README.md index d948263b..874e243e 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -1,10 +1,21 @@ # Docker CLI plugin for Amazon ECS +This was announced at AWS Cloud Containers Conference 2020, read the +[blog post](https://www.docker.com/blog/https-docker-com-blog-from-docker-straight-to-aws/). + ## Status :exclamation: The Docker ECS plugin is still in Beta. Its design and UX will evolve until 1.0 Final release. +## Get started + +If you're using macOS or Windows you just need to install +[Docker Desktop Edge](https://www.docker.com/products/docker-desktop) and you +will have the ECS integration installed. + +You can find Linux install instructions [here](./docs/get-started-linux.md). + ## Example and documentation You can find an application for testing this in [example](./example). From 55d560badba92ae0d03a38f7dab18f4a823151ff Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 9 Jul 2020 18:27:41 +0200 Subject: [PATCH 156/205] readme: Blog link fix Signed-off-by: Christopher Crone Signed-off-by: Nicolas De Loof --- ecs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/README.md b/ecs/README.md index 874e243e..66e419c6 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -1,7 +1,7 @@ # Docker CLI plugin for Amazon ECS This was announced at AWS Cloud Containers Conference 2020, read the -[blog post](https://www.docker.com/blog/https-docker-com-blog-from-docker-straight-to-aws/). +[blog post](https://www.docker.com/blog/from-docker-straight-to-aws/). ## Status From a0a785f19e60bc410c2adb2a4c341c514e9e4877 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 10 Jul 2020 13:54:59 +0200 Subject: [PATCH 157/205] Clarify example/README about required secret NAME Signed-off-by: Nicolas De Loof --- ecs/example/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ecs/example/README.md b/ecs/example/README.md index 96e3e526..e8dbbd73 100644 --- a/ecs/example/README.md +++ b/ecs/example/README.md @@ -36,8 +36,11 @@ If you have 2FA enabled on your Hub account you will need to create a PAT. You can read more about managing access tokens here: https://docs.docker.com/docker-hub/access-tokens/ + +You can then create `DockerHubToken` secret on [AWS Secret Manager](https://aws.amazon.com/secrets-manager/) using following command + ```console -docker ecs secret create -d MyKey -u myhubusername -p myhubpat +docker ecs secret create -d MyKey -u myhubusername -p myhubpat DockerHubToken ``` ### Create an AWS Docker context and list available contexts From 1783716a6aa8b7c20eba394cde6a8af4c9246682 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 10 Jul 2020 09:35:32 +0200 Subject: [PATCH 158/205] claim support for deploy.replicas Signed-off-by: Nicolas De Loof --- ecs/go.mod | 2 +- ecs/go.sum | 2 ++ ecs/pkg/amazon/backend/cloudformation_test.go | 14 ++++++++++++++ ecs/pkg/amazon/backend/compatibility.go | 2 ++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ecs/go.mod b/ecs/go.mod index 332a4688..25611f03 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -14,7 +14,7 @@ require ( github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a + github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 diff --git a/ecs/go.sum b/ecs/go.sum index 9fa33dd9..996e1bb7 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -60,6 +60,8 @@ github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9 h1:WkFqc6U github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a h1:pIiSz5jML7rQ1aupg/KHlTqCxhyXvIgeDMf4kDTzIg8= github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= +github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 h1:F+YIkKDMHdgZBacawhFY1P9RAIgO+6uv2te6hjsjzF0= +github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 518c7c77..8a03e0f5 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -93,6 +93,20 @@ services: assert.Check(t, len(lb.SecurityGroups) > 0) } +func TestServiceReplicas(t *testing.T) { + template := convertYaml(t, ` +version: "3" +services: + test: + image: nginx + deploy: + replicas: 10 +`) + s := template.Resources["TestService"].(*ecs.Service) + assert.Check(t, s != nil) + assert.Check(t, s.DesiredCount == 10) +} + func TestLoadBalancerTypeNetwork(t *testing.T) { template := convertYaml(t, ` version: "3" diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/pkg/amazon/backend/compatibility.go index bc74bd87..85a00128 100644 --- a/ecs/pkg/amazon/backend/compatibility.go +++ b/ecs/pkg/amazon/backend/compatibility.go @@ -14,6 +14,8 @@ var compatibleComposeAttributes = []string{ "services.container_name", "services.cap_drop", "services.depends_on", + "services.deploy", + "services.deploy.replicas", "services.entrypoint", "services.environment", "service.image", From 794ea3cc24092793aff7d16eb6ced6e2000cce6c Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 10 Jul 2020 09:22:38 +0200 Subject: [PATCH 159/205] Check context created by `context` command Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/setup.go | 11 ++++++++ ecs/cmd/commands/setup_test.go | 32 ------------------------ ecs/cmd/commands/testdata/context.golden | 1 - ecs/pkg/amazon/backend/context.go | 24 ++++++++++++++++++ ecs/pkg/amazon/sdk/api.go | 2 ++ ecs/pkg/amazon/sdk/sdk.go | 28 +++++++++++++++------ ecs/pkg/compose/api.go | 2 ++ 7 files changed, 60 insertions(+), 40 deletions(-) delete mode 100644 ecs/cmd/commands/setup_test.go delete mode 100644 ecs/cmd/commands/testdata/context.golden create mode 100644 ecs/pkg/amazon/backend/context.go diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index be876d66..33cd3129 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -1,6 +1,7 @@ package commands import ( + "context" "fmt" "os" "path/filepath" @@ -10,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/defaults" + amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" contextStore "github.com/docker/ecs-plugin/pkg/docker" "github.com/manifoldco/promptui" "github.com/spf13/cobra" @@ -53,6 +55,14 @@ func SetupCommand() *cobra.Command { return err } } + backend, err := amazon.NewBackend(opts.context.Profile, opts.context.Cluster, opts.context.Region) + if err != nil { + return err + } + _, _, err = backend.CreateContextData(context.Background(), nil) + if err != nil { + return err + } return contextStore.NewContext(opts.name, &opts.context) }, } @@ -206,6 +216,7 @@ func setCluster(opts *setupOptions, err error) error { if err != nil { return err } + opts.context.Cluster = result return nil } diff --git a/ecs/cmd/commands/setup_test.go b/ecs/cmd/commands/setup_test.go deleted file mode 100644 index 97b7add3..00000000 --- a/ecs/cmd/commands/setup_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package commands - -import ( - "io/ioutil" - "path/filepath" - "testing" - - "github.com/docker/cli/cli/config" - "gotest.tools/v3/assert" - "gotest.tools/v3/fs" - "gotest.tools/v3/golden" -) - -func TestDefaultAwsContextName(t *testing.T) { - dir := fs.NewDir(t, "setup") - defer dir.Remove() - cmd := NewRootCmd(nil) - dockerConfig := config.Dir() - config.SetDir(dir.Path()) - defer config.SetDir(dockerConfig) - - cmd.SetArgs([]string{"setup", "--cluster", "clusterName", "--profile", "profileName", "--region", "regionName"}) - err := cmd.Execute() - assert.NilError(t, err) - - files, err := filepath.Glob(dir.Join("contexts", "meta", "*", "meta.json")) - assert.NilError(t, err) - assert.Equal(t, len(files), 1) - b, err := ioutil.ReadFile(files[0]) - assert.NilError(t, err) - golden.Assert(t, string(b), "context.golden") -} diff --git a/ecs/cmd/commands/testdata/context.golden b/ecs/cmd/commands/testdata/context.golden deleted file mode 100644 index 891cb9cf..00000000 --- a/ecs/cmd/commands/testdata/context.golden +++ /dev/null @@ -1 +0,0 @@ -{"Name":"aws","Metadata":{"Type":"aws"},"Endpoints":{"aws":{"Profile":"profileName","Cluster":"clusterName","Region":"regionName"},"docker":{"Profile":"profileName","Cluster":"clusterName","Region":"regionName"}}} \ No newline at end of file diff --git a/ecs/pkg/amazon/backend/context.go b/ecs/pkg/amazon/backend/context.go new file mode 100644 index 00000000..bc785d91 --- /dev/null +++ b/ecs/pkg/amazon/backend/context.go @@ -0,0 +1,24 @@ +package backend + +import ( + "context" + "fmt" +) + +func (b *Backend) CreateContextData(ctx context.Context, params map[string]string) (contextData interface{}, description string, err error) { + err = b.api.CheckRequirements(ctx) + if err != nil { + return "", "", err + } + + if b.Cluster != "" { + exists, err := b.api.ClusterExists(ctx, b.Cluster) + if err != nil { + return "", "", err + } + if !exists { + return "", "", fmt.Errorf("cluster %s does not exists", b.Cluster) + } + } + return "", "", nil +} diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index 057823c5..5de5d5af 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -9,6 +9,8 @@ import ( ) type API interface { + CheckRequirements(ctx context.Context) error + GetDefaultVPC(ctx context.Context) (string, error) VpcExists(ctx context.Context, vpcID string) (bool, error) GetSubNets(ctx context.Context, vpcID string) ([]string, error) diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 3365b07f..fd0e649f 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -2,15 +2,13 @@ package sdk import ( "context" + "errors" "fmt" "strings" "time" - "github.com/docker/ecs-plugin/internal" - - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" @@ -27,6 +25,7 @@ import ( "github.com/aws/aws-sdk-go/service/secretsmanager" "github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface" cf "github.com/awslabs/goformation/v4/cloudformation" + "github.com/docker/ecs-plugin/internal" "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" ) @@ -57,8 +56,23 @@ func NewAPI(sess *session.Session) API { } } +func (s sdk) CheckRequirements(ctx context.Context) error { + settings, err := s.ECS.ListAccountSettingsWithContext(ctx, &ecs.ListAccountSettingsInput{ + EffectiveSettings: aws.Bool(true), + Name: aws.String("serviceLongArnFormat"), + }) + if err != nil { + return err + } + serviceLongArnFormat := settings.Settings[0].Value + if *serviceLongArnFormat != "enabled" { + return errors.New("this tool requires the \"new ARN resource ID format\"") + } + return nil +} + func (s sdk) ClusterExists(ctx context.Context, name string) (bool, error) { - logrus.Debug("Check if cluster was already created: ", name) + logrus.Debug("CheckRequirements if cluster was already created: ", name) clusters, err := s.ECS.DescribeClustersWithContext(ctx, &ecs.DescribeClustersInput{ Clusters: []*string{aws.String(name)}, }) @@ -78,7 +92,7 @@ func (s sdk) CreateCluster(ctx context.Context, name string) (string, error) { } func (s sdk) VpcExists(ctx context.Context, vpcID string) (bool, error) { - logrus.Debug("Check if VPC exists: ", vpcID) + logrus.Debug("CheckRequirements if VPC exists: ", vpcID) _, err := s.EC2.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{VpcIds: []*string{&vpcID}}) return err == nil, err } @@ -421,7 +435,7 @@ func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string } func (s sdk) LoadBalancerExists(ctx context.Context, arn string) (bool, error) { - logrus.Debug("Check if LoadBalancer exists: ", arn) + logrus.Debug("CheckRequirements if LoadBalancer exists: ", arn) lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{ LoadBalancerArns: []*string{aws.String(arn)}, }) diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 5f8278c6..66364703 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -12,6 +12,8 @@ type API interface { Up(ctx context.Context, options cli.ProjectOptions) error Down(ctx context.Context, options cli.ProjectOptions) error + CreateContextData(ctx context.Context, params map[string]string) (contextData interface{}, description string, err error) + Convert(project *types.Project) (*cloudformation.Template, error) Logs(ctx context.Context, projectName cli.ProjectOptions) error Ps(background context.Context, options cli.ProjectOptions) ([]ServiceStatus, error) From b7d0b704e5ebd4f7a0872f60f8e0a51cdae0d2ab Mon Sep 17 00:00:00 2001 From: Chad Metcalf Date: Mon, 13 Jul 2020 23:08:36 -0700 Subject: [PATCH 160/205] Change default context name to 'ecs'. The ACI backend uses 'aci' as the default context name. The ECS backend uses 'aws'. There may be other AWS or Azure backends so lets name them for what they are. Addresses issue #154. Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index 33cd3129..c6ac612c 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -66,7 +66,7 @@ func SetupCommand() *cobra.Command { return contextStore.NewContext(opts.name, &opts.context) }, } - cmd.Flags().StringVarP(&opts.name, "name", "n", "aws", "Context Name") + cmd.Flags().StringVarP(&opts.name, "name", "n", "ecs", "Context Name") cmd.Flags().StringVarP(&opts.context.Profile, "profile", "p", "", "AWS Profile") cmd.Flags().StringVarP(&opts.context.Cluster, "cluster", "c", "", "ECS cluster") cmd.Flags().StringVarP(&opts.context.Region, "region", "r", "", "AWS region") From 8ab544a7703dbc447623a3dd06d3c92616ec76da Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 15 Jul 2020 09:52:49 +0200 Subject: [PATCH 161/205] Use env variables from os for interpolation Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 9f5a3841..551b5370 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -46,7 +46,8 @@ func ConvertCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.C cmd := &cobra.Command{ Use: "convert", RunE: docker.WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { - project, err := cli.ProjectFromOptions(options) + opts := options.WithOsEnv() + project, err := cli.ProjectFromOptions(&opts) if err != nil { return err } From b2a90197000c298c0dc7a1a2f3bac009795ef329 Mon Sep 17 00:00:00 2001 From: Chad Metcalf Date: Tue, 14 Jul 2020 22:50:40 -0700 Subject: [PATCH 162/205] LoadBalancer names cannot be longer then 32 characters.. Longer names will fail with: AmazonElasticLoadBalancingV2; Status Code: 400; Error Code: ValidationError; Fairly straight forward approach, truncate at 32 characters. Considering that we append "LoadBalancer" to every name and it is 12 characters the name should stay useful. We can decide later if "LB" is better then a truncated "LoadBalan". The golden test data needed to be updated because its names were also over 32 characters. Fixes #160. Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 4 +++- ecs/pkg/amazon/backend/cloudformation_test.go | 21 ++++++++++--------- ...formation-with-overrides-conversion.golden | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index bf8ecb3f..3cd72360 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -218,7 +218,9 @@ func getLoadBalancerSecurityGroups(project *types.Project, template *cloudformat } func createLoadBalancer(project *types.Project, template *cloudformation.Template) string { - loadBalancerName := fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name)) + + // load balancer names are limited to 32 characters total + loadBalancerName := fmt.Sprintf("%.32s", fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name))) // Create LoadBalancer if `ParameterLoadBalancerName` is not set template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 8a03e0f5..d362a93f 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -34,7 +34,7 @@ func TestSimpleWithOverrides(t *testing.T) { } func TestRolePolicy(t *testing.T) { - template := convertYaml(t, ` + template := convertYaml(t, "test", ` version: "3" services: foo: @@ -54,14 +54,14 @@ services: } func TestMapNetworksToSecurityGroups(t *testing.T) { - template := convertYaml(t, ` + template := convertYaml(t, "test", ` version: "3" services: test: image: hello_world networks: - - front-tier - - back-tier + - front-tier + - back-tier networks: front-tier: @@ -79,7 +79,7 @@ networks: } func TestLoadBalancerTypeApplication(t *testing.T) { - template := convertYaml(t, ` + template := convertYaml(t, "test123456789009876543211234567890", ` version: "3" services: test: @@ -89,12 +89,13 @@ services: `) lb := template.Resources["TestLoadBalancer"].(*elasticloadbalancingv2.LoadBalancer) assert.Check(t, lb != nil) + assert.Check(t, len(lb.Name) <= 32) assert.Check(t, lb.Type == elbv2.LoadBalancerTypeEnumApplication) assert.Check(t, len(lb.SecurityGroups) > 0) } func TestServiceReplicas(t *testing.T) { - template := convertYaml(t, ` + template := convertYaml(t, "test", ` version: "3" services: test: @@ -108,7 +109,7 @@ services: } func TestLoadBalancerTypeNetwork(t *testing.T) { - template := convertYaml(t, ` + template := convertYaml(t, "test", ` version: "3" services: test: @@ -123,7 +124,7 @@ services: } func TestServiceMapping(t *testing.T) { - template := convertYaml(t, ` + template := convertYaml(t, "test", ` version: "3" services: test: @@ -163,7 +164,7 @@ func get(l []ecs.TaskDefinition_KeyValuePair, name string) string { } func TestResourcesHaveProjectTagSet(t *testing.T) { - template := convertYaml(t, ` + template := convertYaml(t, "test", ` version: "3" services: test: @@ -207,7 +208,7 @@ func load(t *testing.T, paths ...string) *types.Project { return project } -func convertYaml(t *testing.T, yaml string) *cloudformation.Template { +func convertYaml(t *testing.T, name string, yaml string) *cloudformation.Template { dict, err := loader.ParseYAML([]byte(yaml)) assert.NilError(t, err) model, err := loader.Load(types.ConfigDetails{ diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden index b2d1ffad..0bc56cf9 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden @@ -260,10 +260,10 @@ }, "Type": "AWS::EC2::SecurityGroupIngress" }, - "TestSimpleWithOverridesLoadBalancer": { + "TestSimpleWithOverridesLoadBalan": { "Condition": "CreateLoadBalancer", "Properties": { - "Name": "TestSimpleWithOverridesLoadBalancer", + "Name": "TestSimpleWithOverridesLoadBalan", "Scheme": "internet-facing", "SecurityGroups": [ { From dbbd24d270857e618c395ee494746be8a4953344 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 15 Jul 2020 18:02:57 +0200 Subject: [PATCH 163/205] Don't create a LoadBalancer if compose app has no port exposed Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 27 +- ecs/pkg/amazon/backend/cloudformation_test.go | 23 +- .../simple-single-service-with-overrides.yaml | 4 - .../testdata/input/simple-single-service.yaml | 4 +- .../simple-cloudformation-conversion.golden | 80 ++++- ...formation-with-overrides-conversion.golden | 292 ------------------ 6 files changed, 117 insertions(+), 313 deletions(-) delete mode 100644 ecs/pkg/amazon/backend/testdata/input/simple-single-service-with-overrides.yaml delete mode 100644 ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 3cd72360..c07f7766 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -142,14 +142,16 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro protocol = elbv2.ProtocolEnumHttp } } - targetGroupName := createTargetGroup(project, service, port, template, protocol) - listenerName := createListener(service, port, template, targetGroupName, loadBalancerARN, protocol) - dependsOn = append(dependsOn, listenerName) - serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ - ContainerName: service.Name, - ContainerPort: int(port.Target), - TargetGroupArn: cloudformation.Ref(targetGroupName), - }) + if loadBalancerARN != "" { + targetGroupName := createTargetGroup(project, service, port, template, protocol) + listenerName := createListener(service, port, template, targetGroupName, loadBalancerARN, protocol) + dependsOn = append(dependsOn, listenerName) + serviceLB = append(serviceLB, ecs.Service_LoadBalancer{ + ContainerName: service.Name, + ContainerPort: int(port.Target), + TargetGroupArn: cloudformation.Ref(targetGroupName), + }) + } } } @@ -218,6 +220,15 @@ func getLoadBalancerSecurityGroups(project *types.Project, template *cloudformat } func createLoadBalancer(project *types.Project, template *cloudformation.Template) string { + ports := 0 + for _, service := range project.Services { + ports += len(service.Ports) + } + if ports == 0 { + // Project do not expose any port (batch jobs?) + // So no need to create a LoadBalancer + return "" + } // load balancer names are limited to 32 characters total loadBalancerName := fmt.Sprintf("%.32s", fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name))) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index d362a93f..032b9cd5 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -26,13 +26,6 @@ func TestSimpleConvert(t *testing.T) { golden.Assert(t, result, expected) } -func TestSimpleWithOverrides(t *testing.T) { - project := load(t, "testdata/input/simple-single-service.yaml", "testdata/input/simple-single-service-with-overrides.yaml") - result := convertResultAsString(t, project, "TestCluster") - expected := "simple/simple-cloudformation-with-overrides-conversion.golden" - golden.Assert(t, result, expected) -} - func TestRolePolicy(t *testing.T) { template := convertYaml(t, "test", ` version: "3" @@ -94,6 +87,22 @@ services: assert.Check(t, len(lb.SecurityGroups) > 0) } +func TestNoLoadBalancerIfNoPortExposed(t *testing.T) { + template := convertYaml(t, "test", ` +version: "3" +services: + test: + image: nginx + foo: + image: bar +`) + for _, r := range template.Resources { + assert.Check(t, r.AWSCloudFormationType() != "AWS::ElasticLoadBalancingV2::TargetGroup") + assert.Check(t, r.AWSCloudFormationType() != "AWS::ElasticLoadBalancingV2::Listener") + assert.Check(t, r.AWSCloudFormationType() != "AWS::ElasticLoadBalancingV2::LoadBalancer") + } +} + func TestServiceReplicas(t *testing.T) { template := convertYaml(t, "test", ` version: "3" diff --git a/ecs/pkg/amazon/backend/testdata/input/simple-single-service-with-overrides.yaml b/ecs/pkg/amazon/backend/testdata/input/simple-single-service-with-overrides.yaml deleted file mode 100644 index 3dc8a0b6..00000000 --- a/ecs/pkg/amazon/backend/testdata/input/simple-single-service-with-overrides.yaml +++ /dev/null @@ -1,4 +0,0 @@ -version: "3" -services: - simple: - image: haproxy diff --git a/ecs/pkg/amazon/backend/testdata/input/simple-single-service.yaml b/ecs/pkg/amazon/backend/testdata/input/simple-single-service.yaml index 4b3f9af2..448f2110 100644 --- a/ecs/pkg/amazon/backend/testdata/input/simple-single-service.yaml +++ b/ecs/pkg/amazon/backend/testdata/input/simple-single-service.yaml @@ -1,4 +1,6 @@ version: "3" services: simple: - image: nginx \ No newline at end of file + image: nginx + ports: + - "80:80" \ No newline at end of file diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden index 0256a586..7e885905 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden @@ -71,6 +71,9 @@ "Type": "AWS::Logs::LogGroup" }, "SimpleService": { + "DependsOn": [ + "SimpleTCP80Listener" + ], "Properties": { "Cluster": { "Fn::If": [ @@ -85,6 +88,15 @@ }, "DesiredCount": 1, "LaunchType": "FARGATE", + "LoadBalancers": [ + { + "ContainerName": "simple", + "ContainerPort": 80, + "TargetGroupArn": { + "Ref": "SimpleTCP80TargetGroup" + } + } + ], "NetworkConfiguration": { "AwsvpcConfiguration": { "AssignPublicIp": "ENABLED", @@ -152,6 +164,56 @@ }, "Type": "AWS::ServiceDiscovery::Service" }, + "SimpleTCP80Listener": { + "Properties": { + "DefaultActions": [ + { + "ForwardConfig": { + "TargetGroups": [ + { + "TargetGroupArn": { + "Ref": "SimpleTCP80TargetGroup" + } + } + ] + }, + "Type": "forward" + } + ], + "LoadBalancerArn": { + "Fn::If": [ + "CreateLoadBalancer", + { + "Ref": "TestSimpleConvertLoadBalancer" + }, + { + "Ref": "ParameterLoadBalancerARN" + } + ] + }, + "Port": 80, + "Protocol": "HTTP" + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener" + }, + "SimpleTCP80TargetGroup": { + "Properties": { + "Name": "SimpleTCP80TargetGroup", + "Port": 80, + "Protocol": "HTTP", + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleConvert" + } + ], + "TargetType": "ip", + "VpcId": { + "Ref": "ParameterVPCId" + } + }, + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup" + }, "SimpleTaskDefinition": { "Properties": { "ContainerDefinitions": [ @@ -188,7 +250,14 @@ "awslogs-stream-prefix": "TestSimpleConvert" } }, - "Name": "simple" + "Name": "simple", + "PortMappings": [ + { + "ContainerPort": 80, + "HostPort": 80, + "Protocol": "tcp" + } + ] } ], "Cpu": "256", @@ -231,6 +300,15 @@ "Properties": { "GroupDescription": "TestSimpleConvert default Security Group", "GroupName": "TestSimpleConvertDefaultNetwork", + "SecurityGroupIngress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "simple:80/tcp", + "FromPort": 80, + "IpProtocol": "TCP", + "ToPort": 80 + } + ], "Tags": [ { "Key": "com.docker.compose.project", diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden deleted file mode 100644 index 0bc56cf9..00000000 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-with-overrides-conversion.golden +++ /dev/null @@ -1,292 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Conditions": { - "CreateCluster": { - "Fn::Equals": [ - "", - { - "Ref": "ParameterClusterName" - } - ] - }, - "CreateLoadBalancer": { - "Fn::Equals": [ - "", - { - "Ref": "ParameterLoadBalancerARN" - } - ] - } - }, - "Parameters": { - "ParameterClusterName": { - "Description": "Name of the ECS cluster to deploy to (optional)", - "Type": "String" - }, - "ParameterLoadBalancerARN": { - "Description": "Name of the LoadBalancer to connect to (optional)", - "Type": "String" - }, - "ParameterSubnet1Id": { - "Description": "SubnetId, for Availability Zone 1 in the region in your VPC", - "Type": "AWS::EC2::Subnet::Id" - }, - "ParameterSubnet2Id": { - "Description": "SubnetId, for Availability Zone 2 in the region in your VPC", - "Type": "AWS::EC2::Subnet::Id" - }, - "ParameterVPCId": { - "Description": "ID of the VPC", - "Type": "AWS::EC2::VPC::Id" - } - }, - "Resources": { - "CloudMap": { - "Properties": { - "Description": "Service Map for Docker Compose project TestSimpleWithOverrides", - "Name": "TestSimpleWithOverrides.local", - "Vpc": { - "Ref": "ParameterVPCId" - } - }, - "Type": "AWS::ServiceDiscovery::PrivateDnsNamespace" - }, - "Cluster": { - "Condition": "CreateCluster", - "Properties": { - "ClusterName": "TestSimpleWithOverrides", - "Tags": [ - { - "Key": "com.docker.compose.project", - "Value": "TestSimpleWithOverrides" - } - ] - }, - "Type": "AWS::ECS::Cluster" - }, - "LogGroup": { - "Properties": { - "LogGroupName": "/docker-compose/TestSimpleWithOverrides" - }, - "Type": "AWS::Logs::LogGroup" - }, - "SimpleService": { - "Properties": { - "Cluster": { - "Fn::If": [ - "CreateCluster", - { - "Ref": "Cluster" - }, - { - "Ref": "ParameterClusterName" - } - ] - }, - "DesiredCount": 1, - "LaunchType": "FARGATE", - "NetworkConfiguration": { - "AwsvpcConfiguration": { - "AssignPublicIp": "ENABLED", - "SecurityGroups": [ - { - "Ref": "TestSimpleWithOverridesDefaultNetwork" - } - ], - "Subnets": [ - { - "Ref": "ParameterSubnet1Id" - }, - { - "Ref": "ParameterSubnet2Id" - } - ] - } - }, - "SchedulingStrategy": "REPLICA", - "ServiceRegistries": [ - { - "RegistryArn": { - "Fn::GetAtt": [ - "SimpleServiceDiscoveryEntry", - "Arn" - ] - } - } - ], - "Tags": [ - { - "Key": "com.docker.compose.project", - "Value": "TestSimpleWithOverrides" - }, - { - "Key": "com.docker.compose.service", - "Value": "simple" - } - ], - "TaskDefinition": { - "Ref": "SimpleTaskDefinition" - } - }, - "Type": "AWS::ECS::Service" - }, - "SimpleServiceDiscoveryEntry": { - "Properties": { - "Description": "\"simple\" service discovery entry in Cloud Map", - "DnsConfig": { - "DnsRecords": [ - { - "TTL": 60, - "Type": "A" - } - ], - "RoutingPolicy": "MULTIVALUE" - }, - "HealthCheckCustomConfig": { - "FailureThreshold": 1 - }, - "Name": "simple", - "NamespaceId": { - "Ref": "CloudMap" - } - }, - "Type": "AWS::ServiceDiscovery::Service" - }, - "SimpleTaskDefinition": { - "Properties": { - "ContainerDefinitions": [ - { - "Environment": [ - { - "Name": "LOCALDOMAIN", - "Value": { - "Fn::Join": [ - "", - [ - { - "Ref": "AWS::Region" - }, - ".compute.internal", - " TestSimpleWithOverrides.local" - ] - ] - } - } - ], - "Essential": true, - "Image": "haproxy", - "LinuxParameters": {}, - "LogConfiguration": { - "LogDriver": "awslogs", - "Options": { - "awslogs-group": { - "Ref": "LogGroup" - }, - "awslogs-region": { - "Ref": "AWS::Region" - }, - "awslogs-stream-prefix": "TestSimpleWithOverrides" - } - }, - "Name": "simple" - } - ], - "Cpu": "256", - "ExecutionRoleArn": { - "Ref": "SimpleTaskExecutionRole" - }, - "Family": "TestSimpleWithOverrides-simple", - "Memory": "512", - "NetworkMode": "awsvpc", - "RequiresCompatibilities": [ - "FARGATE" - ] - }, - "Type": "AWS::ECS::TaskDefinition" - }, - "SimpleTaskExecutionRole": { - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": [ - "sts:AssumeRole" - ], - "Effect": "Allow", - "Principal": { - "Service": "ecs-tasks.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ - "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" - ] - }, - "Type": "AWS::IAM::Role" - }, - "TestSimpleWithOverridesDefaultNetwork": { - "Properties": { - "GroupDescription": "TestSimpleWithOverrides default Security Group", - "GroupName": "TestSimpleWithOverridesDefaultNetwork", - "Tags": [ - { - "Key": "com.docker.compose.project", - "Value": "TestSimpleWithOverrides" - }, - { - "Key": "com.docker.compose.network", - "Value": "default" - } - ], - "VpcId": { - "Ref": "ParameterVPCId" - } - }, - "Type": "AWS::EC2::SecurityGroup" - }, - "TestSimpleWithOverridesDefaultNetworkIngress": { - "Properties": { - "Description": "Allow communication within network default", - "GroupId": { - "Ref": "TestSimpleWithOverridesDefaultNetwork" - }, - "IpProtocol": "-1", - "SourceSecurityGroupId": { - "Ref": "TestSimpleWithOverridesDefaultNetwork" - } - }, - "Type": "AWS::EC2::SecurityGroupIngress" - }, - "TestSimpleWithOverridesLoadBalan": { - "Condition": "CreateLoadBalancer", - "Properties": { - "Name": "TestSimpleWithOverridesLoadBalan", - "Scheme": "internet-facing", - "SecurityGroups": [ - { - "Ref": "TestSimpleWithOverridesDefaultNetwork" - } - ], - "Subnets": [ - { - "Ref": "ParameterSubnet1Id" - }, - { - "Ref": "ParameterSubnet2Id" - } - ], - "Tags": [ - { - "Key": "com.docker.compose.project", - "Value": "TestSimpleWithOverrides" - } - ], - "Type": "application" - }, - "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" - } - } -} From 1b53dbf84fed4dc71087fbadcfd5e34d3152aaaa Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 16 Jul 2020 15:19:28 +0200 Subject: [PATCH 164/205] readme: Update Docker Desktop links Makes it easier to find the Edge build. Signed-off-by: Christopher Crone Signed-off-by: Nicolas De Loof --- ecs/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ecs/README.md b/ecs/README.md index 66e419c6..5faeb696 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -11,10 +11,12 @@ Its design and UX will evolve until 1.0 Final release. ## Get started If you're using macOS or Windows you just need to install -[Docker Desktop Edge](https://www.docker.com/products/docker-desktop) and you -will have the ECS integration installed. +Docker Desktop **Edge** and you will have the ECS integration installed. +You can find links here: +- [macOS](https://hub.docker.com/editions/community/docker-ce-desktop-mac) +- [Windows](https://hub.docker.com/editions/community/docker-ce-desktop-windows) -You can find Linux install instructions [here](./docs/get-started-linux.md). +Linux install instructions are [here](./docs/get-started-linux.md). ## Example and documentation From d7d5e6305406c3b268d712b6d9de84f36e19755e Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 15 Jul 2020 11:42:16 +0200 Subject: [PATCH 165/205] Introduce x-aws-cluster for cluster to deploy application to Signed-off-by: Nicolas De Loof --- ecs/Makefile | 5 +++- ecs/pkg/amazon/backend/list.go | 19 ++++++++------- ecs/pkg/amazon/backend/up.go | 43 ++++++++++++++++++++++++---------- ecs/pkg/compose/x.go | 1 + 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/ecs/Makefile b/ecs/Makefile index cf9c48cc..10932586 100644 --- a/ecs/Makefile +++ b/ecs/Makefile @@ -39,7 +39,10 @@ dev: build lint: ## Verify Go files @docker build . --target lint +fmt: ## Format go files + go fmt ./... + clean: rm -rf dist/ -.PHONY: clean build test dev lint e2e cross +.PHONY: clean build test dev lint e2e cross fmt diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index 232b2c81..8647b0d4 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -14,25 +14,28 @@ func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose return nil, err } - cluster := b.Cluster - if cluster == "" { - cluster = project.Name - } - resources, err := b.api.ListStackResources(ctx, project.Name) if err != nil { return nil, err } - var loadBalancer string - if lb, ok := project.Extensions[compose.ExtensionLB]; ok { - loadBalancer = lb.(string) + loadBalancer, err := b.GetLoadBalancer(ctx, project) + if err != nil { + return nil, err } + + cluster, err := b.GetCluster(ctx, project) + if err != nil { + return nil, err + } + servicesARN := []string{} for _, r := range resources { switch r.Type { case "AWS::ECS::Service": servicesARN = append(servicesARN, r.ARN) + case "AWS::ECS::Cluster": + cluster = r.ARN case "AWS::ElasticLoadBalancingV2::LoadBalancer": loadBalancer = r.ARN } diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 104fef91..e91f1191 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -16,14 +16,14 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { return err } - if b.Cluster != "" { - ok, err := b.api.ClusterExists(ctx, b.Cluster) - if err != nil { - return err - } - if !ok { - return fmt.Errorf("configured cluster %q does not exist", b.Cluster) - } + err = b.api.CheckRequirements(ctx) + if err != nil { + return err + } + + cluster, err := b.GetCluster(ctx, project) + if err != nil { + return err } update, err := b.api.StackExists(ctx, project.Name) @@ -55,7 +55,7 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { } parameters := map[string]string{ - ParameterClusterName: b.Cluster, + ParameterClusterName: cluster, ParameterVPCId: vpc, ParameterSubnet1Id: subNets[0], ParameterSubnet2Id: subNets[1], @@ -96,15 +96,32 @@ func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, er func (b Backend) GetLoadBalancer(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if lb, ok := project.Extensions[compose.ExtensionLB]; ok { - lbName := lb.(string) - ok, err := b.api.LoadBalancerExists(ctx, lbName) + if ext, ok := project.Extensions[compose.ExtensionLB]; ok { + lb := ext.(string) + ok, err := b.api.LoadBalancerExists(ctx, lb) if err != nil { return "", err } if !ok { - return "", fmt.Errorf("Load Balancer does not exist: %s", lb) + return "", fmt.Errorf("load balancer does not exist: %s", lb) } + return lb, nil } return "", nil } + +func (b Backend) GetCluster(ctx context.Context, project *types.Project) (string, error) { + //check compose file for custom VPC selected + if ext, ok := project.Extensions[compose.ExtensionCluster]; ok { + cluster := ext.(string) + ok, err := b.api.ClusterExists(ctx, cluster) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("cluster does not exist: %s", cluster) + } + return cluster, nil + } + return b.Cluster, nil +} diff --git a/ecs/pkg/compose/x.go b/ecs/pkg/compose/x.go index 7a2a5bd0..8e52a368 100644 --- a/ecs/pkg/compose/x.go +++ b/ecs/pkg/compose/x.go @@ -5,4 +5,5 @@ const ( ExtensionVPC = "x-aws-vpc" ExtensionPullCredentials = "x-aws-pull_credentials" ExtensionLB = "x-aws-loadbalancer" + ExtensionCluster = "x-aws-cluster" ) From 37b9e74308a62254deef402433b9730fb4e4b23d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 15 Jul 2020 16:27:24 +0200 Subject: [PATCH 166/205] Implement `ps` without need for the original compose.yaml file close #165 Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/down.go | 22 ++++++++++++++------- ecs/pkg/amazon/backend/list.go | 35 +++++++++++++++++++--------------- ecs/pkg/amazon/sdk/api.go | 1 + ecs/pkg/amazon/sdk/sdk.go | 15 +++++++++++++++ 4 files changed, 51 insertions(+), 22 deletions(-) diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index fcfbc9ac..45a7f844 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -9,13 +9,9 @@ import ( ) func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { - name := options.Name - if name == "" { - project, err := cli.ProjectFromOptions(&options) - if err != nil { - return err - } - name = project.Name + name, err2 := b.projectName(options) + if err2 != nil { + return err2 } err := b.api.DeleteStack(ctx, name) @@ -30,3 +26,15 @@ func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { } return nil } + +func (b *Backend) projectName(options cli.ProjectOptions) (string, error) { + name := options.Name + if name == "" { + project, err := cli.ProjectFromOptions(&options) + if err != nil { + return "", err + } + name = project.Name + } + return name, nil +} diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index 8647b0d4..ff3c4b2a 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -3,33 +3,34 @@ package backend import ( "context" "fmt" + "regexp" + "strings" "github.com/compose-spec/compose-go/cli" "github.com/docker/ecs-plugin/pkg/compose" ) +var targetGroupLogicalName = regexp.MustCompile("(.*)(TCP|UDP)([0-9]+)TargetGroup") + func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose.ServiceStatus, error) { - project, err := cli.ProjectFromOptions(&options) + projectName, err := b.projectName(options) if err != nil { return nil, err } - - resources, err := b.api.ListStackResources(ctx, project.Name) + parameters, err := b.api.ListStackParameters(ctx, projectName) if err != nil { return nil, err } + loadBalancer := parameters[ParameterLoadBalancerARN] + cluster := parameters[ParameterClusterName] - loadBalancer, err := b.GetLoadBalancer(ctx, project) - if err != nil { - return nil, err - } - - cluster, err := b.GetCluster(ctx, project) + resources, err := b.api.ListStackResources(ctx, projectName) if err != nil { return nil, err } servicesARN := []string{} + targetGroups := []string{} for _, r := range resources { switch r.Type { case "AWS::ECS::Service": @@ -38,9 +39,14 @@ func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose cluster = r.ARN case "AWS::ElasticLoadBalancingV2::LoadBalancer": loadBalancer = r.ARN + case "AWS::ElasticLoadBalancingV2::TargetGroup": + targetGroups = append(targetGroups, r.LogicalID) } } + if len(servicesARN) == 0 { + return nil, nil + } status, err := b.api.DescribeServices(ctx, cluster, servicesARN) if err != nil { return nil, err @@ -52,13 +58,12 @@ func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose } for i, state := range status { - s, err := project.GetService(state.Name) - if err != nil { - return nil, err - } ports := []string{} - for _, p := range s.Ports { - ports = append(ports, fmt.Sprintf("%s:%d->%d/%s", url, p.Published, p.Target, p.Protocol)) + for _, tg := range targetGroups { + groups := targetGroupLogicalName.FindStringSubmatch(tg) + if groups[0] == state.Name { + ports = append(ports, fmt.Sprintf("%s:%s->%s/%s", url, groups[2], groups[2], strings.ToLower(groups[1]))) + } } state.Ports = ports status[i] = state diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index 5de5d5af..bce65b29 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -18,6 +18,7 @@ type API interface { StackExists(ctx context.Context, name string) (bool, error) CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error DeleteStack(ctx context.Context, name string) error + ListStackParameters(ctx context.Context, name string) (map[string]string, error) ListStackResources(ctx context.Context, name string) ([]compose.StackResource, error) GetStackID(ctx context.Context, name string) (string, error) WaitStackComplete(ctx context.Context, name string, operation int) error diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index fd0e649f..2af1f78f 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -238,6 +238,21 @@ func (s sdk) DescribeStackEvents(ctx context.Context, stackID string) ([]*cloudf } } +func (s sdk) ListStackParameters(ctx context.Context, name string) (map[string]string, error) { + st, err := s.CF.DescribeStacksWithContext(ctx, &cloudformation.DescribeStacksInput{ + NextToken: nil, + StackName: aws.String(name), + }) + if err != nil { + return nil, err + } + parameters := map[string]string{} + for _, parameter := range st.Stacks[0].Parameters { + parameters[*parameter.ParameterKey] = *parameter.ParameterValue + } + return parameters, nil +} + func (s sdk) ListStackResources(ctx context.Context, name string) ([]compose.StackResource, error) { // FIXME handle pagination res, err := s.CF.ListStackResourcesWithContext(ctx, &cloudformation.ListStackResourcesInput{ From efeded2670a8ae45ae8dbe0dc2fd58889f120ca8 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 16 Jul 2020 08:28:17 +0200 Subject: [PATCH 167/205] Remove Cluster from context Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 10 ++-- ecs/cmd/commands/context.go | 31 +++++++++++ ecs/cmd/commands/secret.go | 8 +-- ecs/cmd/commands/setup.go | 52 ++++++++----------- ecs/pkg/amazon/backend/backend.go | 12 ++--- ecs/pkg/amazon/backend/cloudformation_test.go | 8 +-- ecs/pkg/amazon/backend/context.go | 22 ++++---- ecs/pkg/amazon/backend/down.go | 8 +-- ecs/pkg/amazon/backend/up.go | 2 +- ecs/pkg/docker/contextStore.go | 30 ++--------- ecs/tests/main_test.go | 2 +- 11 files changed, 90 insertions(+), 95 deletions(-) create mode 100644 ecs/cmd/commands/context.go diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 551b5370..bceba59b 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -45,7 +45,7 @@ func (o upOptions) LoadBalancerArn() *string { func ConvertCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "convert", - RunE: docker.WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { opts := options.WithOsEnv() project, err := cli.ProjectFromOptions(&opts) if err != nil { @@ -72,7 +72,7 @@ func UpCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Comman opts := upOptions{} cmd := &cobra.Command{ Use: "up", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { return backend.Up(context.Background(), *options) }), } @@ -84,7 +84,7 @@ func PsCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Comman opts := upOptions{} cmd := &cobra.Command{ Use: "ps", - RunE: docker.WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { status, err := backend.Ps(context.Background(), *options) if err != nil { return err @@ -109,7 +109,7 @@ func DownCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra. opts := downOptions{} cmd := &cobra.Command{ Use: "down", - RunE: docker.WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { return backend.Down(context.Background(), *projectOpts) }), } @@ -120,7 +120,7 @@ func DownCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra. func LogsCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { cmd := &cobra.Command{ Use: "logs [PROJECT NAME]", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { return backend.Logs(context.Background(), *projectOpts) }), } diff --git a/ecs/cmd/commands/context.go b/ecs/cmd/commands/context.go new file mode 100644 index 00000000..080d3b7d --- /dev/null +++ b/ecs/cmd/commands/context.go @@ -0,0 +1,31 @@ +package commands + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/docker/cli/cli/command" + amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" + "github.com/docker/ecs-plugin/pkg/docker" + "github.com/spf13/cobra" +) + +type ContextFunc func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error + +func WithAwsContext(dockerCli command.Cli, f ContextFunc) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, args []string) error { + ctx, err := docker.GetAwsContext(dockerCli) + if err != nil { + return err + } + backend, err := amazon.NewBackend(ctx.Profile, ctx.Region) + if err != nil { + return err + } + err = f(*ctx, backend, args) + if e, ok := err.(awserr.Error); ok { + return fmt.Errorf(e.Message()) + } + return err + } +} diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go index 5bbefaa9..3246e123 100644 --- a/ecs/cmd/commands/secret.go +++ b/ecs/cmd/commands/secret.go @@ -47,7 +47,7 @@ func CreateSecret(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "create NAME", Short: "Creates a secret.", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { if len(args) == 0 { return errors.New("Missing mandatory parameter: NAME") } @@ -69,7 +69,7 @@ func InspectSecret(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "inspect ID", Short: "Displays secret details", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { if len(args) == 0 { return errors.New("Missing mandatory parameter: ID") } @@ -94,7 +94,7 @@ func ListSecrets(dockerCli command.Cli) *cobra.Command { Use: "list", Aliases: []string{"ls"}, Short: "List secrets stored for the existing account.", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { secrets, err := backend.ListSecrets(context.Background()) if err != nil { return err @@ -113,7 +113,7 @@ func DeleteSecret(dockerCli command.Cli) *cobra.Command { Use: "delete NAME", Aliases: []string{"rm", "remove"}, Short: "Removes a secret.", - RunE: docker.WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { if len(args) == 0 { return errors.New("Missing mandatory parameter: [NAME]") } diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go index c6ac612c..da733f71 100644 --- a/ecs/cmd/commands/setup.go +++ b/ecs/cmd/commands/setup.go @@ -22,17 +22,18 @@ const enterLabelPrefix = "Enter " type setupOptions struct { name string - context contextStore.AwsContext + profile string + region string accessKeyID string secretAccessKey string } func (s setupOptions) unsetRequiredArgs() []string { unset := []string{} - if s.context.Profile == "" { + if s.profile == "" { unset = append(unset, "profile") } - if s.context.Region == "" { + if s.region == "" { unset = append(unset, "region") } return unset @@ -51,25 +52,28 @@ func SetupCommand() *cobra.Command { } } if opts.accessKeyID != "" && opts.secretAccessKey != "" { - if err := saveCredentials(opts.context.Profile, opts.accessKeyID, opts.secretAccessKey); err != nil { + if err := saveCredentials(opts.profile, opts.accessKeyID, opts.secretAccessKey); err != nil { return err } } - backend, err := amazon.NewBackend(opts.context.Profile, opts.context.Cluster, opts.context.Region) + backend, err := amazon.NewBackend(opts.profile, opts.region) if err != nil { return err } - _, _, err = backend.CreateContextData(context.Background(), nil) + + context, _, err := backend.CreateContextData(context.Background(), map[string]string{ + amazon.ContextParamProfile: opts.profile, + amazon.ContextParamRegion: opts.region, + }) if err != nil { return err } - return contextStore.NewContext(opts.name, &opts.context) + return contextStore.NewContext(opts.name, context) }, } cmd.Flags().StringVarP(&opts.name, "name", "n", "ecs", "Context Name") - cmd.Flags().StringVarP(&opts.context.Profile, "profile", "p", "", "AWS Profile") - cmd.Flags().StringVarP(&opts.context.Cluster, "cluster", "c", "", "ECS cluster") - cmd.Flags().StringVarP(&opts.context.Region, "region", "r", "", "AWS region") + cmd.Flags().StringVarP(&opts.profile, "profile", "p", "", "AWS Profile") + cmd.Flags().StringVarP(&opts.region, "region", "r", "", "AWS region") cmd.Flags().StringVarP(&opts.accessKeyID, "aws-key-id", "k", "", "AWS Access Key ID") cmd.Flags().StringVarP(&opts.secretAccessKey, "aws-secret-key", "s", "", "AWS Secret Access Key") @@ -88,10 +92,6 @@ func interactiveCli(opts *setupOptions) error { return err } - if err := setCluster(opts, err); err != nil { - return err - } - if err := setRegion(opts, section); err != nil { return err } @@ -156,7 +156,7 @@ func awsProfiles(filename string) (map[string]ini.Section, error) { } func setContextName(opts *setupOptions) error { - if opts.name == "aws" { + if opts.name == "ecs" { result, err := promptString(opts.name, "context name", enterLabelPrefix, 2) if err != nil { return err @@ -171,7 +171,7 @@ func setProfile(opts *setupOptions, section ini.Section) (ini.Section, error) { if err != nil { return ini.Section{}, err } - section, ok := profilesList[opts.context.Profile] + section, ok := profilesList[opts.profile] if !ok { prompt := promptui.Select{ Label: "Select AWS Profile", @@ -179,14 +179,14 @@ func setProfile(opts *setupOptions, section ini.Section) (ini.Section, error) { } _, result, err := prompt.Run() if result == "new profile" { - result, err := promptString(opts.context.Profile, "profile name", enterLabelPrefix, 2) + result, err := promptString(opts.profile, "profile name", enterLabelPrefix, 2) if err != nil { return ini.Section{}, err } - opts.context.Profile = result + opts.profile = result } else { section = profilesList[result] - opts.context.Profile = result + opts.profile = result } if err != nil { return ini.Section{}, err @@ -196,7 +196,7 @@ func setProfile(opts *setupOptions, section ini.Section) (ini.Section, error) { } func setRegion(opts *setupOptions, section ini.Section) error { - defaultRegion := opts.context.Region + defaultRegion := opts.region if defaultRegion == "" && section.Name() != "" { region, err := section.GetKey("region") if err == nil { @@ -207,17 +207,7 @@ func setRegion(opts *setupOptions, section ini.Section) error { if err != nil { return err } - opts.context.Region = result - return nil -} - -func setCluster(opts *setupOptions, err error) error { - result, err := promptString(opts.context.Cluster, "cluster name", enterLabelPrefix, 0) - if err != nil { - return err - } - - opts.context.Cluster = result + opts.region = result return nil } diff --git a/ecs/pkg/amazon/backend/backend.go b/ecs/pkg/amazon/backend/backend.go index 07764df3..f5bba4ed 100644 --- a/ecs/pkg/amazon/backend/backend.go +++ b/ecs/pkg/amazon/backend/backend.go @@ -6,7 +6,7 @@ import ( "github.com/docker/ecs-plugin/pkg/amazon/sdk" ) -func NewBackend(profile string, cluster string, region string) (*Backend, error) { +func NewBackend(profile string, region string) (*Backend, error) { sess, err := session.NewSessionWithOptions(session.Options{ Profile: profile, Config: aws.Config{ @@ -17,14 +17,12 @@ func NewBackend(profile string, cluster string, region string) (*Backend, error) return nil, err } return &Backend{ - Cluster: cluster, - Region: region, - api: sdk.NewAPI(sess), + Region: region, + api: sdk.NewAPI(sess), }, nil } type Backend struct { - Cluster string - Region string - api sdk.API + Region string + api sdk.API } diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 032b9cd5..a364bfaf 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -21,7 +21,7 @@ import ( func TestSimpleConvert(t *testing.T) { project := load(t, "testdata/input/simple-single-service.yaml") - result := convertResultAsString(t, project, "TestCluster") + result := convertResultAsString(t, project) expected := "simple/simple-cloudformation-conversion.golden" golden.Assert(t, result, expected) } @@ -197,10 +197,10 @@ services: } } -func convertResultAsString(t *testing.T, project *types.Project, clusterName string) string { - client, err := NewBackend("", clusterName, "") +func convertResultAsString(t *testing.T, project *types.Project) string { + backend, err := NewBackend("", "") assert.NilError(t, err) - result, err := client.Convert(project) + result, err := backend.Convert(project) assert.NilError(t, err) resultAsJSON, err := result.JSON() assert.NilError(t, err) diff --git a/ecs/pkg/amazon/backend/context.go b/ecs/pkg/amazon/backend/context.go index bc785d91..f6f79f4e 100644 --- a/ecs/pkg/amazon/backend/context.go +++ b/ecs/pkg/amazon/backend/context.go @@ -2,7 +2,13 @@ package backend import ( "context" - "fmt" + + "github.com/docker/ecs-plugin/pkg/docker" +) + +const ( + ContextParamRegion = "region" + ContextParamProfile = "profile" ) func (b *Backend) CreateContextData(ctx context.Context, params map[string]string) (contextData interface{}, description string, err error) { @@ -11,14 +17,8 @@ func (b *Backend) CreateContextData(ctx context.Context, params map[string]strin return "", "", err } - if b.Cluster != "" { - exists, err := b.api.ClusterExists(ctx, b.Cluster) - if err != nil { - return "", "", err - } - if !exists { - return "", "", fmt.Errorf("cluster %s does not exists", b.Cluster) - } - } - return "", "", nil + return docker.AwsContext{ + Profile: params[ContextParamProfile], + Region: params[ContextParamRegion], + }, "Amazon ECS context", nil } diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index 45a7f844..31df94fc 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -9,12 +9,12 @@ import ( ) func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { - name, err2 := b.projectName(options) - if err2 != nil { - return err2 + name, err := b.projectName(options) + if err != nil { + return err } - err := b.api.DeleteStack(ctx, name) + err = b.api.DeleteStack(ctx, name) if err != nil { return err } diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index e91f1191..8b975dc5 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -123,5 +123,5 @@ func (b Backend) GetCluster(ctx context.Context, project *types.Project) (string } return cluster, nil } - return b.Cluster, nil + return "", nil } diff --git a/ecs/pkg/docker/contextStore.go b/ecs/pkg/docker/contextStore.go index 83c055f6..30a12de2 100644 --- a/ecs/pkg/docker/contextStore.go +++ b/ecs/pkg/docker/contextStore.go @@ -3,13 +3,10 @@ package docker import ( "fmt" - "github.com/aws/aws-sdk-go/aws/awserr" "github.com/docker/cli/cli/command" cliconfig "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/context/store" - amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" "github.com/mitchellh/mapstructure" - "github.com/spf13/cobra" ) const contextType = "aws" @@ -24,16 +21,15 @@ func getter() interface{} { type AwsContext struct { Profile string - Cluster string Region string } -func NewContext(name string, awsContext *AwsContext) error { - _, err := NewContextWithStore(name, awsContext, cliconfig.ContextStoreDir()) +func NewContext(name string, awsContext interface{}) error { + _, err := NewContextWithStore(name, awsContext.(AwsContext), cliconfig.ContextStoreDir()) return err } -func NewContextWithStore(name string, awsContext *AwsContext, contextDirectory string) (store.Store, error) { +func NewContextWithStore(name string, awsContext AwsContext, contextDirectory string) (store.Store, error) { contextStore := initContextStore(contextDirectory) endpoints := map[string]interface{}{ "aws": awsContext, @@ -74,26 +70,6 @@ func checkAwsContextExists(contextName string) (*AwsContext, error) { return &awsContext, nil } -type ContextFunc func(ctx AwsContext, backend *amazon.Backend, args []string) error - -func WithAwsContext(dockerCli command.Cli, f ContextFunc) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - ctx, err := GetAwsContext(dockerCli) - if err != nil { - return err - } - backend, err := amazon.NewBackend(ctx.Profile, ctx.Cluster, ctx.Region) - if err != nil { - return err - } - err = f(*ctx, backend, args) - if e, ok := err.(awserr.Error); ok { - return fmt.Errorf(e.Message()) - } - return err - } -} - func GetAwsContext(dockerCli command.Cli) (*AwsContext, error) { contextName := dockerCli.CurrentContext() return checkAwsContextExists(contextName) diff --git a/ecs/tests/main_test.go b/ecs/tests/main_test.go index b8a75814..5849e406 100644 --- a/ecs/tests/main_test.go +++ b/ecs/tests/main_test.go @@ -58,7 +58,7 @@ func (d dockerCliCommand) createTestCmd(ops ...ConfigFileOperator) (icmd.Cmd, fu Profile: "sandbox.devtools.developer", Region: "eu-west-3", } - testStore, err := docker.NewContextWithStore(testContextName, &awsContext, filepath.Join(configDir, "contexts")) + testStore, err := docker.NewContextWithStore(testContextName, awsContext, filepath.Join(configDir, "contexts")) if err != nil { panic(err) } From 2d931dab9dac20002cc80d357fe465e577dfd53e Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 16 Jul 2020 16:07:26 +0200 Subject: [PATCH 168/205] `up` can update an existing stack using CloudFormation Changeset Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/up.go | 29 ++++++++++++------ ecs/pkg/amazon/sdk/api.go | 2 ++ ecs/pkg/amazon/sdk/sdk.go | 59 ++++++++++++++++++++++++++++++++++-- ecs/pkg/compose/types.go | 1 + 4 files changed, 78 insertions(+), 13 deletions(-) diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 8b975dc5..3ea21d17 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -26,14 +26,6 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { return err } - update, err := b.api.StackExists(ctx, project.Name) - if err != nil { - return err - } - if update { - return fmt.Errorf("we do not (yet) support updating an existing CloudFormation stack") - } - template, err := b.Convert(project) if err != nil { return err @@ -62,17 +54,34 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { ParameterLoadBalancerARN: lb, } - err = b.api.CreateStack(ctx, project.Name, template, parameters) + update, err := b.api.StackExists(ctx, project.Name) if err != nil { return err } + operation := compose.StackCreate + if update { + operation = compose.StackUpdate + changeset, err := b.api.CreateChangeSet(ctx, project.Name, template, parameters) + if err != nil { + return err + } + err = b.api.UpdateStack(ctx, changeset) + if err != nil { + return err + } + } else { + err = b.api.CreateStack(ctx, project.Name, template, parameters) + if err != nil { + return err + } + } fmt.Println() w := console.NewProgressWriter() for k := range template.Resources { w.ResourceEvent(k, "PENDING", "") } - return b.WaitStackCompletion(ctx, project.Name, compose.StackCreate, w) + return b.WaitStackCompletion(ctx, project.Name, operation, w) } func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, error) { diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index bce65b29..c56df268 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -23,6 +23,8 @@ type API interface { GetStackID(ctx context.Context, name string) (string, error) WaitStackComplete(ctx context.Context, name string, operation int) error DescribeStackEvents(ctx context.Context, stackID string) ([]*cf.StackEvent, error) + CreateChangeSet(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) (string, error) + UpdateStack(ctx context.Context, changeset string) error DescribeServices(ctx context.Context, cluster string, arns []string) ([]compose.ServiceStatus, error) diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 2af1f78f..b2d0cb15 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -175,9 +175,8 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template param := []*cloudformation.Parameter{} for name, value := range parameters { param = append(param, &cloudformation.Parameter{ - ParameterKey: aws.String(name), - ParameterValue: aws.String(value), - UsePreviousValue: aws.Bool(true), + ParameterKey: aws.String(name), + ParameterValue: aws.String(value), }) } @@ -194,6 +193,60 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template return err } +func (s sdk) CreateChangeSet(ctx context.Context, name string, template *cf.Template, parameters map[string]string) (string, error) { + logrus.Debug("Create CloudFormation Changeset") + json, err := template.JSON() + if err != nil { + return "", err + } + + param := []*cloudformation.Parameter{} + for name := range parameters { + param = append(param, &cloudformation.Parameter{ + ParameterKey: aws.String(name), + UsePreviousValue: aws.Bool(true), + }) + } + + update := fmt.Sprintf("Update%s", time.Now().Format("2006-01-02-15-04-05")) + changeset, err := s.CF.CreateChangeSetWithContext(ctx, &cloudformation.CreateChangeSetInput{ + ChangeSetName: aws.String(update), + ChangeSetType: aws.String(cloudformation.ChangeSetTypeUpdate), + StackName: aws.String(name), + TemplateBody: aws.String(string(json)), + Parameters: param, + Capabilities: []*string{ + aws.String(cloudformation.CapabilityCapabilityIam), + }, + }) + if err != nil { + return "", err + } + + err = s.CF.WaitUntilChangeSetCreateCompleteWithContext(ctx, &cloudformation.DescribeChangeSetInput{ + ChangeSetName: changeset.Id, + }) + return *changeset.Id, err +} + +func (s sdk) UpdateStack(ctx context.Context, changeset string) error { + desc, err := s.CF.DescribeChangeSetWithContext(ctx, &cloudformation.DescribeChangeSetInput{ + ChangeSetName: aws.String(changeset), + }) + if err != nil { + return err + } + + if strings.HasPrefix(aws.StringValue(desc.StatusReason), "The submitted information didn't contain changes.") { + return nil + } + + _, err = s.CF.ExecuteChangeSet(&cloudformation.ExecuteChangeSetInput{ + ChangeSetName: aws.String(changeset), + }) + return err +} + func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) error { input := &cloudformation.DescribeStacksInput{ StackName: aws.String(name), diff --git a/ecs/pkg/compose/types.go b/ecs/pkg/compose/types.go index 370bfc4b..ec59578a 100644 --- a/ecs/pkg/compose/types.go +++ b/ecs/pkg/compose/types.go @@ -19,6 +19,7 @@ type ServiceStatus struct { const ( StackCreate = iota + StackUpdate StackDelete ) From 12215130b575f82c7d508ff33c97ee9572e8e341 Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 21 Jul 2020 17:45:16 +0200 Subject: [PATCH 169/205] generic URL/port/protocol retrieval for compose ps Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 2 +- ecs/pkg/amazon/backend/list.go | 24 ++++-------- ecs/pkg/amazon/sdk/sdk.go | 70 +++++++++++++++++++++++++++++++--- ecs/pkg/compose/types.go | 23 ++++++++--- 4 files changed, 91 insertions(+), 28 deletions(-) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index bceba59b..75a9555f 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -91,7 +91,7 @@ func PsCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Comman } printSection(os.Stdout, len(status), func(w io.Writer) { for _, service := range status { - fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\n", service.ID, service.Name, service.Replicas, service.Desired, strings.Join(service.Ports, " ")) + fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\n", service.ID, service.Name, service.Replicas, service.Desired, strings.Join(service.Ports, ", ")) } }, "ID", "NAME", "REPLICAS", "PORTS") return nil diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index ff3c4b2a..165241da 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -21,7 +21,6 @@ func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose if err != nil { return nil, err } - loadBalancer := parameters[ParameterLoadBalancerARN] cluster := parameters[ParameterClusterName] resources, err := b.api.ListStackResources(ctx, projectName) @@ -30,20 +29,14 @@ func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose } servicesARN := []string{} - targetGroups := []string{} for _, r := range resources { switch r.Type { case "AWS::ECS::Service": servicesARN = append(servicesARN, r.ARN) case "AWS::ECS::Cluster": cluster = r.ARN - case "AWS::ElasticLoadBalancingV2::LoadBalancer": - loadBalancer = r.ARN - case "AWS::ElasticLoadBalancingV2::TargetGroup": - targetGroups = append(targetGroups, r.LogicalID) } } - if len(servicesARN) == 0 { return nil, nil } @@ -52,18 +45,15 @@ func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose return nil, err } - url, err := b.api.GetLoadBalancerURL(ctx, loadBalancer) - if err != nil { - return nil, err - } - for i, state := range status { ports := []string{} - for _, tg := range targetGroups { - groups := targetGroupLogicalName.FindStringSubmatch(tg) - if groups[0] == state.Name { - ports = append(ports, fmt.Sprintf("%s:%s->%s/%s", url, groups[2], groups[2], strings.ToLower(groups[1]))) - } + for _, lb := range state.LoadBalancers { + ports = append(ports, fmt.Sprintf( + "%s:%d->%d/%s", + lb.URL, + lb.PublishedPort, + lb.TargetPort, + strings.ToLower(lb.Protocol))) } state.Ports = ports status[i] = state diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index b2d0cb15..7447ecc9 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -449,6 +449,7 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, arns []string if err != nil { return nil, err } + status := []compose.ServiceStatus{} for _, service := range services.Services { var name string @@ -460,17 +461,76 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, arns []string if name == "" { return nil, fmt.Errorf("service %s doesn't have a %s tag", *service.ServiceArn, compose.ServiceTag) } + targetGroupArns := []string{} + for _, lb := range service.LoadBalancers { + targetGroupArns = append(targetGroupArns, *lb.TargetGroupArn) + } + // getURLwithPortMapping makes 2 queries + // one to get the target groups and another for load balancers + loadBalancers, err := s.getURLWithPortMapping(ctx, targetGroupArns) + if err != nil { + return nil, err + } status = append(status, compose.ServiceStatus{ - ID: *service.ServiceName, - Name: name, - Replicas: int(*service.RunningCount), - Desired: int(*service.DesiredCount), + ID: *service.ServiceName, + Name: name, + Replicas: int(*service.RunningCount), + Desired: int(*service.DesiredCount), + LoadBalancers: loadBalancers, }) } - return status, nil } +func (s sdk) getURLWithPortMapping(ctx context.Context, targetGroupArns []string) ([]compose.LoadBalancer, error) { + if len(targetGroupArns) == 0 { + return nil, nil + } + groups, err := s.ELB.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{ + TargetGroupArns: aws.StringSlice(targetGroupArns), + }) + if err != nil { + return nil, err + } + lbarns := []*string{} + for _, tg := range groups.TargetGroups { + lbarns = append(lbarns, tg.LoadBalancerArns...) + } + + lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{ + LoadBalancerArns: lbarns, + }) + + if err != nil { + return nil, err + } + filterLB := func(arn *string, lbs []*elbv2.LoadBalancer) *elbv2.LoadBalancer { + for _, lb := range lbs { + if *lb.LoadBalancerArn == *arn { + return lb + } + } + return nil + } + loadBalancers := []compose.LoadBalancer{} + for _, tg := range groups.TargetGroups { + for _, lbarn := range tg.LoadBalancerArns { + lb := filterLB(lbarn, lbs.LoadBalancers) + if lb == nil { + continue + } + loadBalancers = append(loadBalancers, compose.LoadBalancer{ + URL: *lb.DNSName, + TargetPort: int(*tg.Port), + PublishedPort: int(*tg.Port), + Protocol: *tg.Protocol, + }) + + } + } + return loadBalancers, nil +} + func (s sdk) ListTasks(ctx context.Context, cluster string, family string) ([]string, error) { tasks, err := s.ECS.ListTasksWithContext(ctx, &ecs.ListTasksInput{ Cluster: aws.String(cluster), diff --git a/ecs/pkg/compose/types.go b/ecs/pkg/compose/types.go index ec59578a..1426d1be 100644 --- a/ecs/pkg/compose/types.go +++ b/ecs/pkg/compose/types.go @@ -9,12 +9,25 @@ type StackResource struct { Status string } +type PortMapping struct { + Source int + Target int +} + +type LoadBalancer struct { + URL string + TargetPort int + PublishedPort int + Protocol string +} + type ServiceStatus struct { - ID string - Name string - Replicas int - Desired int - Ports []string + ID string + Name string + Replicas int + Desired int + Ports []string + LoadBalancers []LoadBalancer } const ( From 3706d8617f058af130576bbaf31a8b7d9ed17362 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 22 Jul 2020 15:48:04 +0200 Subject: [PATCH 170/205] Pointers value access fixes Signed-off-by: aiordache (cherry picked from commit 65d6fe57546b4f7a58ecf878d8740155285bc19a) Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/sdk/sdk.go | 52 ++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 7447ecc9..3ccb668c 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -301,7 +301,7 @@ func (s sdk) ListStackParameters(ctx context.Context, name string) (map[string]s } parameters := map[string]string{} for _, parameter := range st.Stacks[0].Parameters { - parameters[*parameter.ParameterKey] = *parameter.ParameterValue + parameters[aws.StringValue(parameter.ParameterKey)] = aws.StringValue(parameter.ParameterValue) } return parameters, nil } @@ -318,10 +318,10 @@ func (s sdk) ListStackResources(ctx context.Context, name string) ([]compose.Sta resources := []compose.StackResource{} for _, r := range res.StackResourceSummaries { resources = append(resources, compose.StackResource{ - LogicalID: *r.LogicalResourceId, - Type: *r.ResourceType, - ARN: *r.PhysicalResourceId, - Status: *r.ResourceStatus, + LogicalID: aws.StringValue(r.LogicalResourceId), + Type: aws.StringValue(r.ResourceType), + ARN: aws.StringValue(r.PhysicalResourceId), + Status: aws.StringValue(r.ResourceStatus), }) } return resources, nil @@ -350,7 +350,7 @@ func (s sdk) CreateSecret(ctx context.Context, secret compose.Secret) (string, e if err != nil { return "", err } - return *response.ARN, nil + return aws.StringValue(response.ARN), nil } func (s sdk) InspectSecret(ctx context.Context, id string) (compose.Secret, error) { @@ -361,11 +361,11 @@ func (s sdk) InspectSecret(ctx context.Context, id string) (compose.Secret, erro } labels := map[string]string{} for _, tag := range response.Tags { - labels[*tag.Key] = *tag.Value + labels[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value) } secret := compose.Secret{ - ID: *response.ARN, - Name: *response.Name, + ID: aws.StringValue(response.ARN), + Name: aws.StringValue(response.Name), Labels: labels, } if response.Description != nil { @@ -431,8 +431,8 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer compose.LogConsu } for _, event := range events.Events { - p := strings.Split(*event.LogStreamName, "/") - consumer.Log(p[1], p[2], *event.Message) + p := strings.Split(aws.StringValue(event.LogStreamName), "/") + consumer.Log(p[1], p[2], aws.StringValue(event.Message)) startTime = *event.IngestionTime } } @@ -455,7 +455,7 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, arns []string var name string for _, t := range service.Tags { if *t.Key == compose.ServiceTag { - name = *t.Value + name = aws.StringValue(t.Value) } } if name == "" { @@ -472,10 +472,10 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, arns []string return nil, err } status = append(status, compose.ServiceStatus{ - ID: *service.ServiceName, + ID: aws.StringValue(service.ServiceName), Name: name, - Replicas: int(*service.RunningCount), - Desired: int(*service.DesiredCount), + Replicas: int(aws.Int64Value(service.RunningCount)), + Desired: int(aws.Int64Value(service.DesiredCount)), LoadBalancers: loadBalancers, }) } @@ -505,8 +505,12 @@ func (s sdk) getURLWithPortMapping(ctx context.Context, targetGroupArns []string return nil, err } filterLB := func(arn *string, lbs []*elbv2.LoadBalancer) *elbv2.LoadBalancer { + if aws.StringValue(arn) == "" { + // load balancer arn is nil/"" + return nil + } for _, lb := range lbs { - if *lb.LoadBalancerArn == *arn { + if aws.StringValue(lb.LoadBalancerArn) == aws.StringValue(arn) { return lb } } @@ -520,10 +524,10 @@ func (s sdk) getURLWithPortMapping(ctx context.Context, targetGroupArns []string continue } loadBalancers = append(loadBalancers, compose.LoadBalancer{ - URL: *lb.DNSName, - TargetPort: int(*tg.Port), - PublishedPort: int(*tg.Port), - Protocol: *tg.Protocol, + URL: aws.StringValue(lb.DNSName), + TargetPort: int(aws.Int64Value(tg.Port)), + PublishedPort: int(aws.Int64Value(tg.Port)), + Protocol: aws.StringValue(tg.Protocol), }) } @@ -556,7 +560,7 @@ func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string publicIPs := map[string]string{} for _, interf := range desc.NetworkInterfaces { if interf.Association != nil { - publicIPs[*interf.NetworkInterfaceId] = *interf.Association.PublicIp + publicIPs[aws.StringValue(interf.NetworkInterfaceId)] = aws.StringValue(interf.Association.PublicIp) } } return publicIPs, nil @@ -581,5 +585,9 @@ func (s sdk) GetLoadBalancerURL(ctx context.Context, arn string) (string, error) if err != nil { return "", err } - return *lbs.LoadBalancers[0].DNSName, nil + dnsName := aws.StringValue(lbs.LoadBalancers[0].DNSName) + if dnsName == "" { + return "", fmt.Errorf("Load balancer %s doesn't have a dns name", aws.StringValue(lbs.LoadBalancers[0].LoadBalancerArn)) + } + return dnsName, nil } From 2ef1e28f1d7da2c0dc3c8005e4110f409ba99bcd Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 23 Jul 2020 14:37:05 +0200 Subject: [PATCH 171/205] Add support for deploy.resources Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation_test.go | 38 +++++++++++++++++++ ecs/pkg/amazon/backend/compatibility.go | 6 +++ ecs/pkg/amazon/backend/convert.go | 28 ++++++++++++-- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index a364bfaf..e7efad18 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -117,6 +117,44 @@ services: assert.Check(t, s.DesiredCount == 10) } +func TestTaskSizeConvert(t *testing.T) { + template := convertYaml(t, "test", ` +version: "3" +services: + test: + image: nginx + deploy: + resources: + limits: + cpus: '0.5' + memory: 2048M + reservations: + cpus: '0.5' + memory: 2048M +`) + def := template.Resources["TestTaskDefinition"].(*ecs.TaskDefinition) + assert.Equal(t, def.Cpu, "512") + assert.Equal(t, def.Memory, "2048") + + template = convertYaml(t, "test", ` +version: "3" +services: + test: + image: nginx + deploy: + resources: + limits: + cpus: '4' + memory: 8192M + reservations: + cpus: '4' + memory: 8192M +`) + def = template.Resources["TestTaskDefinition"].(*ecs.TaskDefinition) + assert.Equal(t, def.Cpu, "4096") + assert.Equal(t, def.Memory, "8192") +} + func TestLoadBalancerTypeNetwork(t *testing.T) { template := convertYaml(t, "test", ` version: "3" diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/pkg/amazon/backend/compatibility.go index 85a00128..35d58d83 100644 --- a/ecs/pkg/amazon/backend/compatibility.go +++ b/ecs/pkg/amazon/backend/compatibility.go @@ -16,6 +16,12 @@ var compatibleComposeAttributes = []string{ "services.depends_on", "services.deploy", "services.deploy.replicas", + "services.deploy.resources.limits", + "services.deploy.resources.limits.cpus", + "services.deploy.resources.limits.memory", + "services.deploy.resources.reservations", + "services.deploy.resources.reservations.cpus", + "services.deploy.resources.reservations.memory", "services.entrypoint", "services.environment", "service.image", diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index 6fdc0abc..6bd66255 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -21,6 +21,10 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi if err != nil { return nil, err } + _, memReservation, err := toContainerReservation(service) + if err != nil { + return nil, err + } credential := getRepoCredentials(service) // override resolve.conf search directive to also search .local @@ -60,6 +64,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi "awslogs-stream-prefix": project.Name, }, }, + MemoryReservation: memReservation, Name: service.Name, PortMappings: toPortMappings(service.Ports), Privileged: service.Privileged, @@ -111,6 +116,8 @@ func toSystemControls(sysctls types.Mapping) []ecs.TaskDefinition_SystemControl return sys } +const Mb = 1024 * 1024 + func toLimits(service types.ServiceConfig) (string, string, error) { // All possible cpu/mem values for Fargate cpuToMem := map[int64][]types.UnitBytes{ @@ -142,9 +149,9 @@ func toLimits(service types.ServiceConfig) (string, string, error) { } for cpu, mem := range cpuToMem { - if v <= cpu*1024*1024 { + if v <= cpu*Mb { for _, m := range mem { - if limits.MemoryBytes <= m*1024*1024 { + if limits.MemoryBytes <= m*Mb { cpuLimit = strconv.FormatInt(cpu, 10) memLimit = strconv.FormatInt(int64(m), 10) return cpuLimit, memLimit, nil @@ -155,6 +162,21 @@ func toLimits(service types.ServiceConfig) (string, string, error) { return "", "", fmt.Errorf("unable to find cpu/mem for the required resources") } +func toContainerReservation(service types.ServiceConfig) (string, int, error) { + cpuReservation := ".0" + memReservation := 0 + + if service.Deploy == nil { + return cpuReservation, memReservation, nil + } + + reservations := service.Deploy.Resources.Reservations + if reservations == nil { + return cpuReservation, memReservation, nil + } + return reservations.NanoCPUs, int(reservations.MemoryBytes / Mb), nil +} + func toRequiresCompatibilities(isolation string) []*string { if isolation == "" { return nil @@ -206,8 +228,6 @@ func toUlimits(ulimits map[string]*types.UlimitsConfig) []ecs.TaskDefinition_Uli return u } -const Mb = 1024 * 1024 - func toLinuxParameters(service types.ServiceConfig) *ecs.TaskDefinition_LinuxParameters { return &ecs.TaskDefinition_LinuxParameters{ Capabilities: toKernelCapabilities(service.CapAdd, service.CapDrop), From ce3ab38e6144169351b68e3527cbe26b83b1b0b2 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 23 Jul 2020 20:17:41 +0200 Subject: [PATCH 172/205] Add test for convert failure Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation_test.go | 26 ++++++++++++++++--- ecs/pkg/amazon/backend/convert.go | 8 +++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index e7efad18..c8622a1a 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -154,6 +154,21 @@ services: assert.Equal(t, def.Cpu, "4096") assert.Equal(t, def.Memory, "8192") } +func TestTaskSizeConvertFailure(t *testing.T) { + model := loadConfig(t, "test", ` +version: "3" +services: + test: + image: nginx + deploy: + resources: + limits: + cpus: '0.5' + memory: 2043248M +`) + _, err := Backend{}.Convert(model) + assert.ErrorContains(t, err, "unable to find cpu/mem for the required resources") +} func TestLoadBalancerTypeNetwork(t *testing.T) { template := convertYaml(t, "test", ` @@ -256,6 +271,13 @@ func load(t *testing.T, paths ...string) *types.Project { } func convertYaml(t *testing.T, name string, yaml string) *cloudformation.Template { + model := loadConfig(t, name, yaml) + template, err := Backend{}.Convert(model) + assert.NilError(t, err) + return template +} + +func loadConfig(t *testing.T, name string, yaml string) *types.Project { dict, err := loader.ParseYAML([]byte(yaml)) assert.NilError(t, err) model, err := loader.Load(types.ConfigDetails{ @@ -266,7 +288,5 @@ func convertYaml(t *testing.T, name string, yaml string) *cloudformation.Templat options.Name = "Test" }) assert.NilError(t, err) - template, err := Backend{}.Convert(model) - assert.NilError(t, err) - return template + return model } diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index 6bd66255..828618b4 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -116,7 +116,7 @@ func toSystemControls(sysctls types.Mapping) []ecs.TaskDefinition_SystemControl return sys } -const Mb = 1024 * 1024 +const MiB = 1024 * 1024 func toLimits(service types.ServiceConfig) (string, string, error) { // All possible cpu/mem values for Fargate @@ -149,9 +149,9 @@ func toLimits(service types.ServiceConfig) (string, string, error) { } for cpu, mem := range cpuToMem { - if v <= cpu*Mb { + if v <= cpu*MiB { for _, m := range mem { - if limits.MemoryBytes <= m*Mb { + if limits.MemoryBytes <= m*MiB { cpuLimit = strconv.FormatInt(cpu, 10) memLimit = strconv.FormatInt(int64(m), 10) return cpuLimit, memLimit, nil @@ -174,7 +174,7 @@ func toContainerReservation(service types.ServiceConfig) (string, int, error) { if reservations == nil { return cpuReservation, memReservation, nil } - return reservations.NanoCPUs, int(reservations.MemoryBytes / Mb), nil + return reservations.NanoCPUs, int(reservations.MemoryBytes / MiB), nil } func toRequiresCompatibilities(isolation string) []*string { From 94671e99e131e9bbd86022d389cff2f6d7902867 Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 24 Jul 2020 09:46:52 +0200 Subject: [PATCH 173/205] improve error message for unsupported resource combination Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation_test.go | 2 +- ecs/pkg/amazon/backend/convert.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index c8622a1a..679d3fe6 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -167,7 +167,7 @@ services: memory: 2043248M `) _, err := Backend{}.Convert(model) - assert.ErrorContains(t, err, "unable to find cpu/mem for the required resources") + assert.ErrorContains(t, err, "The resources requested are not supported by ECS/Fargate") } func TestLoadBalancerTypeNetwork(t *testing.T) { diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index 828618b4..49b9b445 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -159,7 +159,7 @@ func toLimits(service types.ServiceConfig) (string, string, error) { } } } - return "", "", fmt.Errorf("unable to find cpu/mem for the required resources") + return "", "", fmt.Errorf("The resources requested are not supported by ECS/Fargate") } func toContainerReservation(service types.ServiceConfig) (string, int, error) { From 716fd1369041bf7480469542bde15020b5b09db3 Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 24 Jul 2020 10:17:48 +0200 Subject: [PATCH 174/205] sort cpu values in conversion to fargate values Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation_test.go | 2 +- ecs/pkg/amazon/backend/convert.go | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 679d3fe6..0c7edd8f 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -167,7 +167,7 @@ services: memory: 2043248M `) _, err := Backend{}.Convert(model) - assert.ErrorContains(t, err, "The resources requested are not supported by ECS/Fargate") + assert.ErrorContains(t, err, "the resources requested are not supported by ECS/Fargate") } func TestLoadBalancerTypeNetwork(t *testing.T) { diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index 49b9b445..e3591ee7 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -2,6 +2,7 @@ package backend import ( "fmt" + "sort" "strconv" "strings" "time" @@ -148,7 +149,14 @@ func toLimits(service types.ServiceConfig) (string, string, error) { return "", "", err } - for cpu, mem := range cpuToMem { + var cpus []int64 + for k := range cpuToMem { + cpus = append(cpus, k) + } + sort.Slice(cpus, func(i, j int) bool { return cpus[i] < cpus[j] }) + + for _, cpu := range cpus { + mem := cpuToMem[cpu] if v <= cpu*MiB { for _, m := range mem { if limits.MemoryBytes <= m*MiB { @@ -159,7 +167,7 @@ func toLimits(service types.ServiceConfig) (string, string, error) { } } } - return "", "", fmt.Errorf("The resources requested are not supported by ECS/Fargate") + return "", "", fmt.Errorf("the resources requested are not supported by ECS/Fargate") } func toContainerReservation(service types.ServiceConfig) (string, int, error) { From 1cde94729771cd4188e9fbcf1dba26794cf4cfb4 Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 24 Jul 2020 17:24:26 +0200 Subject: [PATCH 175/205] fix custom vpc setup Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/Dockerfile | 2 +- ecs/pkg/amazon/backend/up.go | 4 ++++ ecs/pkg/amazon/sdk/sdk.go | 4 ---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ecs/Dockerfile b/ecs/Dockerfile index 185a080e..e5b506b9 100644 --- a/ecs/Dockerfile +++ b/ecs/Dockerfile @@ -1,6 +1,6 @@ # syntax = docker/dockerfile:experimental ARG GO_VERSION=1.14.4-alpine -ARG ALPINE_PKG_DOCKER_VERSION=19.03.11-r0 +ARG ALPINE_PKG_DOCKER_VERSION=19.03.12-r0 ARG GOLANGCI_LINT_VERSION=v1.27.0-alpine FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS base diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 3ea21d17..ee871fe5 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -40,6 +40,9 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { if err != nil { return err } + if len(subNets) < 2 { + return fmt.Errorf("VPC %s should have at least 2 associated subnets in different availability zones", vpc) + } lb, err := b.GetLoadBalancer(ctx, project) if err != nil { @@ -95,6 +98,7 @@ func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, er if !ok { return "", fmt.Errorf("VPC does not exist: %s", vpc) } + return vpcID, nil } defaultVPC, err := b.api.GetDefaultVPC(ctx) if err != nil { diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 3ccb668c..7b5af748 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -125,10 +125,6 @@ func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) { Name: aws.String("vpc-id"), Values: []*string{aws.String(vpcID)}, }, - { - Name: aws.String("default-for-az"), - Values: []*string{aws.String("true")}, - }, }, }) if err != nil { From f5703310ef07bd681ded961ea2d4789d97981474 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 27 Jul 2020 09:44:42 +0200 Subject: [PATCH 176/205] Add template description Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 2 +- .../testdata/simple/simple-cloudformation-conversion.golden | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index c07f7766..2d85a8a7 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -51,7 +51,7 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro } template := cloudformation.NewTemplate() - + template.Description = "CloudFormation template created by Docker for deploying applications on Amazon ECS" template.Parameters[ParameterClusterName] = cloudformation.Parameter{ Type: "String", Description: "Name of the ECS cluster to deploy to (optional)", diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden index 7e885905..4db597e4 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden @@ -18,6 +18,7 @@ ] } }, + "Description": "CloudFormation template created by Docker for deploying applications on Amazon ECS", "Parameters": { "ParameterClusterName": { "Description": "Name of the ECS cluster to deploy to (optional)", From cec3187bbb4a95a23cafe590f7d2a72a3c6bc3ce Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 24 Jul 2020 20:01:17 +0200 Subject: [PATCH 177/205] Set task tags Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/convert.go | 32 ++++++++++++++----- .../simple-cloudformation-conversion.golden | 14 ++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index e3591ee7..4470e116 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -37,14 +37,29 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi fmt.Sprintf(" %s.local", project.Name), })) + tags := []tags.Tag{ + { + Key: compose.ProjectTag, + Value: project.Name, + }, + { + Key: compose.ServiceTag, + Value: service.Name, + }, + } + tags = append(tags, toTags(service.Labels)...) + return &ecs.TaskDefinition{ ContainerDefinitions: []ecs.TaskDefinition_ContainerDefinition{ { - Command: service.Command, - DisableNetworking: service.NetworkMode == "none", - DnsSearchDomains: service.DNSSearch, - DnsServers: service.DNS, - DockerLabels: nil, + Command: service.Command, + DisableNetworking: service.NetworkMode == "none", + DnsSearchDomains: service.DNSSearch, + DnsServers: service.DNS, + DockerLabels: map[string]string{ + compose.ProjectTag: project.Name, + compose.ServiceTag: service.Name, + }, DockerSecurityOptions: service.SecurityOpt, EntryPoint: service.Entrypoint, Environment: toKeyValuePair(service.Environment), @@ -78,8 +93,9 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi SystemControls: toSystemControls(service.Sysctls), Ulimits: toUlimits(service.Ulimits), User: service.User, - VolumesFrom: nil, - WorkingDirectory: service.WorkingDir, + + VolumesFrom: nil, + WorkingDirectory: service.WorkingDir, }, }, Cpu: cpu, @@ -91,7 +107,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi PlacementConstraints: toPlacementConstraints(service.Deploy), ProxyConfiguration: nil, RequiresCompatibilities: []string{ecsapi.LaunchTypeFargate}, - Tags: toTags(service.Labels), + Tags: tags, }, nil } diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden index 4db597e4..2538d4c0 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden @@ -219,6 +219,10 @@ "Properties": { "ContainerDefinitions": [ { + "DockerLabels": { + "com.docker.compose.project": "TestSimpleConvert", + "com.docker.compose.service": "simple" + }, "Environment": [ { "Name": "LOCALDOMAIN", @@ -270,6 +274,16 @@ "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE" + ], + "Tags": [ + { + "Key": "com.docker.compose.project", + "Value": "TestSimpleConvert" + }, + { + "Key": "com.docker.compose.service", + "Value": "simple" + } ] }, "Type": "AWS::ECS::TaskDefinition" From 55531eb6b4cf08b9dc6878cc43174e668d4d3cfa Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 27 Jul 2020 16:23:46 +0200 Subject: [PATCH 178/205] Remove `compose up` timeout Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/up.go | 12 ++++++++++++ ecs/pkg/amazon/sdk/sdk.go | 2 +- ecs/pkg/compose/types.go | 5 ----- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index ee871fe5..d7279416 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -3,6 +3,9 @@ package backend import ( "context" "fmt" + "os" + "os/signal" + "syscall" "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" @@ -84,6 +87,15 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { for k := range template.Resources { w.ResourceEvent(k, "PENDING", "") } + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-signalChan + fmt.Println("user interrupted deployment. Deleting stack...") + b.Down(ctx, options) + }() + return b.WaitStackCompletion(ctx, project.Name, operation, w) } diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 7b5af748..2dcd2ca8 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -181,7 +181,7 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template StackName: aws.String(name), TemplateBody: aws.String(string(json)), Parameters: param, - TimeoutInMinutes: aws.Int64(15), + TimeoutInMinutes: nil, Capabilities: []*string{ aws.String(cloudformation.CapabilityCapabilityIam), }, diff --git a/ecs/pkg/compose/types.go b/ecs/pkg/compose/types.go index 1426d1be..807b9c7a 100644 --- a/ecs/pkg/compose/types.go +++ b/ecs/pkg/compose/types.go @@ -9,11 +9,6 @@ type StackResource struct { Status string } -type PortMapping struct { - Source int - Target int -} - type LoadBalancer struct { URL string TargetPort int From 0df99075cc9ce1d985103dcd6a2b9bb1c0b10589 Mon Sep 17 00:00:00 2001 From: David Killmon Date: Fri, 31 Jul 2020 11:38:41 -0700 Subject: [PATCH 179/205] add shared config state to session By adding this flag to the session, we force the AWS Go SDK to read the ~/.aws/config file. By default, the Go SDK doesn't read this file which is often not what we or customers expect. Many customers store their assume role based prfoiles in the .aws/config file rather than the .aws/credentials file. (This is what the AWS CLI does, by default - but that's because this parameter is enabled by default in the python SDK). Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/backend.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecs/pkg/amazon/backend/backend.go b/ecs/pkg/amazon/backend/backend.go index f5bba4ed..4fd40c2b 100644 --- a/ecs/pkg/amazon/backend/backend.go +++ b/ecs/pkg/amazon/backend/backend.go @@ -8,7 +8,8 @@ import ( func NewBackend(profile string, region string) (*Backend, error) { sess, err := session.NewSessionWithOptions(session.Options{ - Profile: profile, + Profile: profile, + SharedConfigState: session.SharedConfigEnable, Config: aws.Config{ Region: aws.String(region), }, From 9ac3ce772cbeb9ec572785629d048bbdef2cf179 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 3 Aug 2020 17:12:38 +0200 Subject: [PATCH 180/205] Better diagnostic message for "new ARN format" requirement Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/context.go | 15 ++++++++++++--- ecs/pkg/amazon/backend/up.go | 2 +- ecs/pkg/amazon/sdk/api.go | 2 +- ecs/pkg/amazon/sdk/sdk.go | 7 ++++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/ecs/pkg/amazon/backend/context.go b/ecs/pkg/amazon/backend/context.go index f6f79f4e..c1865117 100644 --- a/ecs/pkg/amazon/backend/context.go +++ b/ecs/pkg/amazon/backend/context.go @@ -2,6 +2,7 @@ package backend import ( "context" + "fmt" "github.com/docker/ecs-plugin/pkg/docker" ) @@ -12,13 +13,21 @@ const ( ) func (b *Backend) CreateContextData(ctx context.Context, params map[string]string) (contextData interface{}, description string, err error) { - err = b.api.CheckRequirements(ctx) + region, ok := params[ContextParamRegion] + if !ok { + return nil, "", fmt.Errorf("%q parameter is required", ContextParamRegion) + } + profile, ok := params[ContextParamProfile] + if !ok { + return nil, "", fmt.Errorf("%q parameter is required", ContextParamProfile) + } + err = b.api.CheckRequirements(ctx, region) if err != nil { return "", "", err } return docker.AwsContext{ - Profile: params[ContextParamProfile], - Region: params[ContextParamRegion], + Profile: profile, + Region: region, }, "Amazon ECS context", nil } diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index d7279416..6593f07b 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -19,7 +19,7 @@ func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { return err } - err = b.api.CheckRequirements(ctx) + err = b.api.CheckRequirements(ctx, b.Region) if err != nil { return err } diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go index c56df268..af6dfed4 100644 --- a/ecs/pkg/amazon/sdk/api.go +++ b/ecs/pkg/amazon/sdk/api.go @@ -9,7 +9,7 @@ import ( ) type API interface { - CheckRequirements(ctx context.Context) error + CheckRequirements(ctx context.Context, region string) error GetDefaultVPC(ctx context.Context) (string, error) VpcExists(ctx context.Context, vpcID string) (bool, error) diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index 2dcd2ca8..da40fa52 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -2,7 +2,6 @@ package sdk import ( "context" - "errors" "fmt" "strings" "time" @@ -56,7 +55,7 @@ func NewAPI(sess *session.Session) API { } } -func (s sdk) CheckRequirements(ctx context.Context) error { +func (s sdk) CheckRequirements(ctx context.Context, region string) error { settings, err := s.ECS.ListAccountSettingsWithContext(ctx, &ecs.ListAccountSettingsInput{ EffectiveSettings: aws.Bool(true), Name: aws.String("serviceLongArnFormat"), @@ -66,7 +65,9 @@ func (s sdk) CheckRequirements(ctx context.Context) error { } serviceLongArnFormat := settings.Settings[0].Value if *serviceLongArnFormat != "enabled" { - return errors.New("this tool requires the \"new ARN resource ID format\"") + return fmt.Errorf("this tool requires the \"new ARN resource ID format\".\n"+ + "Check https://%s.console.aws.amazon.com/ecs/home#/settings\n"+ + "Learn more: https://aws.amazon.com/blogs/compute/migrating-your-amazon-ecs-deployment-to-the-new-arn-and-resource-id-format-2", region) } return nil } From 1a3c75fa2905390816de12ccf054b33e190879dd Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 4 Aug 2020 10:23:21 +0200 Subject: [PATCH 181/205] Update aws go sdk and goformation Signed-off-by: Nicolas De Loof --- ecs/go.mod | 5 +++-- ecs/go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ecs/go.mod b/ecs/go.mod index 25611f03..aa1145ba 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -5,8 +5,8 @@ require ( github.com/Microsoft/hcsshim v0.8.7 // indirect github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect - github.com/aws/aws-sdk-go v1.30.22 - github.com/awslabs/goformation/v4 v4.8.0 + github.com/aws/aws-sdk-go v1.33.18 + github.com/awslabs/goformation/v4 v4.14.0 github.com/bitly/go-hostpool v0.1.0 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect @@ -28,6 +28,7 @@ require ( github.com/gogo/protobuf v1.3.1 // indirect github.com/gorilla/mux v1.7.3 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect + github.com/imdario/mergo v0.3.10 // indirect github.com/jinzhu/gorm v1.9.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lib/pq v1.3.0 // indirect diff --git a/ecs/go.sum b/ecs/go.sum index 996e1bb7..2d32a669 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -21,8 +21,12 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.30.22 h1:wImJ8jQrplgmxaTeUY7FrJFn4te/VtWq+mmmJ1TnWAg= github.com/aws/aws-sdk-go v1.30.22/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.33.18 h1:Ccy1SV2SsgJU3rfrD+SOhQ0jvuzfrFuja/oKI86ruPw= +github.com/aws/aws-sdk-go v1.33.18/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/awslabs/goformation/v4 v4.8.0 h1:UiUhyokRy3suEqBXTnipvY8klqY3Eyl4GCH17brraEc= github.com/awslabs/goformation/v4 v4.8.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= +github.com/awslabs/goformation/v4 v4.14.0 h1:E2Pet9eIqA4qzt3dzzzE4YN83V4Kyfbcio0VokBC9TA= +github.com/awslabs/goformation/v4 v4.14.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -160,6 +164,8 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= From 8582cb3928188f7cd36252c9ef3535ef24bcfe21 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 16 Jul 2020 15:25:15 +0200 Subject: [PATCH 182/205] update compose-go and adopt NewProjectOptions and functional parameters Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 47 ++++++++++++++++++++++++---------- ecs/cmd/commands/opts.go | 19 +++++++++++++- ecs/go.mod | 2 +- ecs/go.sum | 2 ++ ecs/pkg/amazon/backend/down.go | 6 ++--- ecs/pkg/amazon/backend/list.go | 5 +--- ecs/pkg/amazon/backend/logs.go | 4 +-- ecs/pkg/amazon/backend/up.go | 4 +-- ecs/pkg/compose/api.go | 8 +++--- 9 files changed, 66 insertions(+), 31 deletions(-) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index 75a9555f..f4262fd4 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -18,7 +18,7 @@ func ComposeCommand(dockerCli command.Cli) *cobra.Command { cmd := &cobra.Command{ Use: "compose", } - opts := &cli.ProjectOptions{} + opts := &composeOptions{} AddFlags(opts, cmd.Flags()) cmd.AddCommand( @@ -42,12 +42,15 @@ func (o upOptions) LoadBalancerArn() *string { return &o.loadBalancerArn } -func ConvertCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Command { +func ConvertCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { cmd := &cobra.Command{ Use: "convert", RunE: WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { - opts := options.WithOsEnv() - project, err := cli.ProjectFromOptions(&opts) + opts, err := options.toProjectOptions() + if err != nil { + return err + } + project, err := cli.ProjectFromOptions(opts) if err != nil { return err } @@ -68,24 +71,32 @@ func ConvertCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.C return cmd } -func UpCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Command { +func UpCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ Use: "up", RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - return backend.Up(context.Background(), *options) + opts, err := options.toProjectOptions() + if err != nil { + return err + } + return backend.Up(context.Background(), opts) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") return cmd } -func PsCommand(dockerCli command.Cli, options *cli.ProjectOptions) *cobra.Command { +func PsCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { opts := upOptions{} cmd := &cobra.Command{ Use: "ps", - RunE: WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { - status, err := backend.Ps(context.Background(), *options) + RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + opts, err := options.toProjectOptions() + if err != nil { + return err + } + status, err := backend.Ps(context.Background(), opts) if err != nil { return err } @@ -105,23 +116,31 @@ type downOptions struct { DeleteCluster bool } -func DownCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { +func DownCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { opts := downOptions{} cmd := &cobra.Command{ Use: "down", - RunE: WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { - return backend.Down(context.Background(), *projectOpts) + RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { + opts, err := options.toProjectOptions() + if err != nil { + return err + } + return backend.Down(context.Background(), opts) }), } cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") return cmd } -func LogsCommand(dockerCli command.Cli, projectOpts *cli.ProjectOptions) *cobra.Command { +func LogsCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { cmd := &cobra.Command{ Use: "logs [PROJECT NAME]", RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - return backend.Logs(context.Background(), *projectOpts) + opts, err := options.toProjectOptions() + if err != nil { + return err + } + return backend.Logs(context.Background(), opts) }), } return cmd diff --git a/ecs/cmd/commands/opts.go b/ecs/cmd/commands/opts.go index bb63ec38..2124ca24 100644 --- a/ecs/cmd/commands/opts.go +++ b/ecs/cmd/commands/opts.go @@ -5,7 +5,24 @@ import ( "github.com/spf13/pflag" ) -func AddFlags(o *cli.ProjectOptions, flags *pflag.FlagSet) { +type composeOptions struct { + Name string + WorkingDir string + ConfigPaths []string + Environment []string +} + +func AddFlags(o *composeOptions, flags *pflag.FlagSet) { flags.StringArrayVarP(&o.ConfigPaths, "file", "f", nil, "Specify an alternate compose file") flags.StringVarP(&o.Name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") + flags.StringVarP(&o.WorkingDir, "workdir", "w", "", "Working directory") + flags.StringSliceVarP(&o.Environment, "environment", "e", []string{}, "Environment variables") +} + +func (o *composeOptions) toProjectOptions() (*cli.ProjectOptions, error) { + return cli.NewProjectOptions(o.ConfigPaths, + cli.WithOsEnv, + cli.WithEnv(o.Environment), + cli.WithWorkingDirectory(o.WorkingDir), + cli.WithName(o.Name)) } diff --git a/ecs/go.mod b/ecs/go.mod index aa1145ba..f3e2949d 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -14,7 +14,7 @@ require ( github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 + github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3 github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 diff --git a/ecs/go.sum b/ecs/go.sum index 2d32a669..296309a3 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -66,6 +66,8 @@ github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a h1:pIiSz5j github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 h1:F+YIkKDMHdgZBacawhFY1P9RAIgO+6uv2te6hjsjzF0= github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= +github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3 h1:+ntlMTrEcScJjlnEOP8P1IIrusJaR93Eazr66YgUueA= +github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index 31df94fc..fe89a9f2 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -8,7 +8,7 @@ import ( "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { +func (b *Backend) Down(ctx context.Context, options *cli.ProjectOptions) error { name, err := b.projectName(options) if err != nil { return err @@ -27,10 +27,10 @@ func (b *Backend) Down(ctx context.Context, options cli.ProjectOptions) error { return nil } -func (b *Backend) projectName(options cli.ProjectOptions) (string, error) { +func (b *Backend) projectName(options *cli.ProjectOptions) (string, error) { name := options.Name if name == "" { - project, err := cli.ProjectFromOptions(&options) + project, err := cli.ProjectFromOptions(options) if err != nil { return "", err } diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/pkg/amazon/backend/list.go index 165241da..59150314 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/pkg/amazon/backend/list.go @@ -3,16 +3,13 @@ package backend import ( "context" "fmt" - "regexp" "strings" "github.com/compose-spec/compose-go/cli" "github.com/docker/ecs-plugin/pkg/compose" ) -var targetGroupLogicalName = regexp.MustCompile("(.*)(TCP|UDP)([0-9]+)TargetGroup") - -func (b *Backend) Ps(ctx context.Context, options cli.ProjectOptions) ([]compose.ServiceStatus, error) { +func (b *Backend) Ps(ctx context.Context, options *cli.ProjectOptions) ([]compose.ServiceStatus, error) { projectName, err := b.projectName(options) if err != nil { return nil, err diff --git a/ecs/pkg/amazon/backend/logs.go b/ecs/pkg/amazon/backend/logs.go index 00b8b2d4..ef5c61e2 100644 --- a/ecs/pkg/amazon/backend/logs.go +++ b/ecs/pkg/amazon/backend/logs.go @@ -13,10 +13,10 @@ import ( "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) Logs(ctx context.Context, options cli.ProjectOptions) error { +func (b *Backend) Logs(ctx context.Context, options *cli.ProjectOptions) error { name := options.Name if name == "" { - project, err := cli.ProjectFromOptions(&options) + project, err := cli.ProjectFromOptions(options) if err != nil { return err } diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 6593f07b..4bbe5bd0 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -13,8 +13,8 @@ import ( "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) Up(ctx context.Context, options cli.ProjectOptions) error { - project, err := cli.ProjectFromOptions(&options) +func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { + project, err := cli.ProjectFromOptions(options) if err != nil { return err } diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 66364703..8e99b8ab 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -9,14 +9,14 @@ import ( ) type API interface { - Up(ctx context.Context, options cli.ProjectOptions) error - Down(ctx context.Context, options cli.ProjectOptions) error + Up(ctx context.Context, options *cli.ProjectOptions) error + Down(ctx context.Context, options *cli.ProjectOptions) error CreateContextData(ctx context.Context, params map[string]string) (contextData interface{}, description string, err error) Convert(project *types.Project) (*cloudformation.Template, error) - Logs(ctx context.Context, projectName cli.ProjectOptions) error - Ps(background context.Context, options cli.ProjectOptions) ([]ServiceStatus, error) + Logs(ctx context.Context, options *cli.ProjectOptions) error + Ps(background context.Context, options *cli.ProjectOptions) ([]ServiceStatus, error) CreateSecret(ctx context.Context, secret Secret) (string, error) InspectSecret(ctx context.Context, id string) (Secret, error) From e7bc8081ba755eeecbba8f8dd5f2b1e1b55cb9c9 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 4 Aug 2020 18:20:34 +0200 Subject: [PATCH 183/205] Propagate service tags on Tasks closes #188 Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 1 + ecs/pkg/amazon/backend/convert.go | 30 ++++--------------- .../simple-cloudformation-conversion.golden | 1 + 3 files changed, 8 insertions(+), 24 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 2d85a8a7..2c6d4bc5 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -179,6 +179,7 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro }, }, }, + PropagateTags: ecsapi.PropagateTagsService, SchedulingStrategy: ecsapi.SchedulingStrategyReplica, ServiceRegistries: []ecs.Service_ServiceRegistry{serviceRegistry}, Tags: []tags.Tag{ diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index 4470e116..b4ae15ae 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -37,29 +37,13 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi fmt.Sprintf(" %s.local", project.Name), })) - tags := []tags.Tag{ - { - Key: compose.ProjectTag, - Value: project.Name, - }, - { - Key: compose.ServiceTag, - Value: service.Name, - }, - } - tags = append(tags, toTags(service.Labels)...) - return &ecs.TaskDefinition{ ContainerDefinitions: []ecs.TaskDefinition_ContainerDefinition{ { - Command: service.Command, - DisableNetworking: service.NetworkMode == "none", - DnsSearchDomains: service.DNSSearch, - DnsServers: service.DNS, - DockerLabels: map[string]string{ - compose.ProjectTag: project.Name, - compose.ServiceTag: service.Name, - }, + Command: service.Command, + DisableNetworking: service.NetworkMode == "none", + DnsSearchDomains: service.DNSSearch, + DnsServers: service.DNS, DockerSecurityOptions: service.SecurityOpt, EntryPoint: service.Entrypoint, Environment: toKeyValuePair(service.Environment), @@ -93,9 +77,8 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi SystemControls: toSystemControls(service.Sysctls), Ulimits: toUlimits(service.Ulimits), User: service.User, - - VolumesFrom: nil, - WorkingDirectory: service.WorkingDir, + VolumesFrom: nil, + WorkingDirectory: service.WorkingDir, }, }, Cpu: cpu, @@ -107,7 +90,6 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi PlacementConstraints: toPlacementConstraints(service.Deploy), ProxyConfiguration: nil, RequiresCompatibilities: []string{ecsapi.LaunchTypeFargate}, - Tags: tags, }, nil } diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden index 2538d4c0..c941d5fe 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden @@ -116,6 +116,7 @@ ] } }, + "PropagateTags": "SERVICE", "SchedulingStrategy": "REPLICA", "ServiceRegistries": [ { From 35019564e43bed2c5e84b4221cd9866fc0ffd241 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 6 Aug 2020 10:22:55 +0200 Subject: [PATCH 184/205] Configure Deployment controller Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 12 +++++++++-- .../simple-cloudformation-conversion.golden | 21 +++++++------------ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 2c6d4bc5..74602318 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -163,12 +163,20 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro for _, dependency := range service.DependsOn { dependsOn = append(dependsOn, serviceResourceName(dependency)) } + template.Resources[serviceResourceName(service.Name)] = &ecs.Service{ AWSCloudFormationDependsOn: dependsOn, Cluster: cluster, DesiredCount: desiredCount, - LaunchType: ecsapi.LaunchTypeFargate, - LoadBalancers: serviceLB, + DeploymentController: &ecs.Service_DeploymentController{ + Type: ecsapi.DeploymentControllerTypeEcs, + }, + DeploymentConfiguration: &ecs.Service_DeploymentConfiguration{ + MaximumPercent: 200, + MinimumHealthyPercent: 100, + }, + LaunchType: ecsapi.LaunchTypeFargate, + LoadBalancers: serviceLB, NetworkConfiguration: &ecs.Service_NetworkConfiguration{ AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: ecsapi.AssignPublicIpEnabled, diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden index c941d5fe..c61fc649 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden @@ -87,6 +87,13 @@ } ] }, + "DeploymentConfiguration": { + "MaximumPercent": 200, + "MinimumHealthyPercent": 100 + }, + "DeploymentController": { + "Type": "ECS" + }, "DesiredCount": 1, "LaunchType": "FARGATE", "LoadBalancers": [ @@ -220,10 +227,6 @@ "Properties": { "ContainerDefinitions": [ { - "DockerLabels": { - "com.docker.compose.project": "TestSimpleConvert", - "com.docker.compose.service": "simple" - }, "Environment": [ { "Name": "LOCALDOMAIN", @@ -275,16 +278,6 @@ "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE" - ], - "Tags": [ - { - "Key": "com.docker.compose.project", - "Value": "TestSimpleConvert" - }, - { - "Key": "com.docker.compose.service", - "Value": "simple" - } ] }, "Type": "AWS::ECS::TaskDefinition" From 59e8eaf744e08e08f83855284454e393d63e5add Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 10 Aug 2020 08:32:52 +0200 Subject: [PATCH 185/205] Don't set targetGroup a name to avoid conflicts Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 74602318..54cebb20 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -307,7 +307,6 @@ func createTargetGroup(project *types.Project, service types.ServiceConfig, port port.Published, ) template.Resources[targetGroupName] = &elasticloadbalancingv2.TargetGroup{ - Name: targetGroupName, Port: int(port.Target), Protocol: protocol, Tags: []tags.Tag{ From b05af0c0accd89f26506c14571ef5dfd8403e9fd Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 10 Aug 2020 08:27:57 +0200 Subject: [PATCH 186/205] Claim support for healthcheck.retries Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/compatibility.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/pkg/amazon/backend/compatibility.go index 35d58d83..93806e96 100644 --- a/ecs/pkg/amazon/backend/compatibility.go +++ b/ecs/pkg/amazon/backend/compatibility.go @@ -28,6 +28,7 @@ var compatibleComposeAttributes = []string{ "services.init", "services.healthcheck", "services.healthcheck.interval", + "services.healthcheck.retries", "services.healthcheck.start_period", "services.healthcheck.test", "services.healthcheck.timeout", From 85b3cbd6ea8f7d6dc88751dd8dbf572d968941bc Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 4 Aug 2020 18:04:06 +0200 Subject: [PATCH 187/205] use an initContainer to inject secrets as /run/secrets/xx Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 6 +- ecs/go.sum | 14 +- ecs/pkg/amazon/backend/cloudformation.go | 27 ++++ ecs/pkg/amazon/backend/compatibility.go | 6 + ecs/pkg/amazon/backend/convert.go | 157 ++++++++++++++++------ ecs/pkg/amazon/cloudformation/marshall.go | 45 +++++++ ecs/pkg/amazon/sdk/sdk.go | 6 +- ecs/pkg/compose/x.go | 1 + ecs/secrets/Dockerfile | 8 ++ ecs/secrets/main.go | 85 ++++++++++++ 10 files changed, 296 insertions(+), 59 deletions(-) create mode 100644 ecs/pkg/amazon/cloudformation/marshall.go create mode 100644 ecs/secrets/Dockerfile create mode 100644 ecs/secrets/main.go diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index f4262fd4..f4e6d4e9 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -7,6 +7,8 @@ import ( "os" "strings" + "github.com/docker/ecs-plugin/pkg/amazon/cloudformation" + "github.com/compose-spec/compose-go/cli" "github.com/docker/cli/cli/command" amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" @@ -59,11 +61,11 @@ func ConvertCommand(dockerCli command.Cli, options *composeOptions) *cobra.Comma return err } - j, err := template.JSON() + json, err := cloudformation.Marshall(template) if err != nil { fmt.Printf("Failed to generate JSON: %s\n", err) } else { - fmt.Printf("%s\n", string(j)) + fmt.Printf("%s\n", string(json)) } return nil }), diff --git a/ecs/go.sum b/ecs/go.sum index 296309a3..96406699 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -19,12 +19,8 @@ github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkK github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.30.22 h1:wImJ8jQrplgmxaTeUY7FrJFn4te/VtWq+mmmJ1TnWAg= -github.com/aws/aws-sdk-go v1.30.22/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.33.18 h1:Ccy1SV2SsgJU3rfrD+SOhQ0jvuzfrFuja/oKI86ruPw= github.com/aws/aws-sdk-go v1.33.18/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/awslabs/goformation/v4 v4.8.0 h1:UiUhyokRy3suEqBXTnipvY8klqY3Eyl4GCH17brraEc= -github.com/awslabs/goformation/v4 v4.8.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= github.com/awslabs/goformation/v4 v4.14.0 h1:E2Pet9eIqA4qzt3dzzzE4YN83V4Kyfbcio0VokBC9TA= github.com/awslabs/goformation/v4 v4.14.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -58,14 +54,6 @@ github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiK github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo= github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= -github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8 h1:sVvKsoXizFOuJNc8dM91IeET2/zDNFj3hwHgk437iJ8= -github.com/compose-spec/compose-go v0.0.0-20200624120600-614475470cd8/go.mod h1:ih9anT8po+49hrb+1j3ldIJ/YRAaBH52ErlQLTKE2Yo= -github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9 h1:WkFqc6UpRqxROso9KC+ceaTiXx/VWpeO1x+NV0d4d+o= -github.com/compose-spec/compose-go v0.0.0-20200707124823-710ff8e60ad9/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= -github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a h1:pIiSz5jML7rQ1aupg/KHlTqCxhyXvIgeDMf4kDTzIg8= -github.com/compose-spec/compose-go v0.0.0-20200709084333-492a50989a5a/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= -github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 h1:F+YIkKDMHdgZBacawhFY1P9RAIgO+6uv2te6hjsjzF0= -github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3 h1:+ntlMTrEcScJjlnEOP8P1IIrusJaR93Eazr66YgUueA= github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= @@ -80,6 +68,7 @@ github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -475,6 +464,7 @@ gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/kubernetes v1.13.0 h1:qTfB+u5M92k2fCCCVP2iuhgwwSOv1EkAkvQY1tQODD8= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 54cebb20..a47e1ad6 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -2,9 +2,12 @@ package backend import ( "fmt" + "io/ioutil" "regexp" "strings" + "github.com/awslabs/goformation/v4/cloudformation/secretsmanager" + ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/elbv2" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" @@ -93,6 +96,30 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro networks[net.Name] = convertNetwork(project, net, cloudformation.Ref(ParameterVPCId), template) } + for i, s := range project.Secrets { + if s.External.External { + continue + } + secret, err := ioutil.ReadFile(s.File) + if err != nil { + return nil, err + } + + name := fmt.Sprintf("%sSecret", normalizeResourceName(s.Name)) + template.Resources[name] = &secretsmanager.Secret{ + Description: "", + SecretString: string(secret), + Tags: []tags.Tag{ + { + Key: compose.ProjectTag, + Value: project.Name, + }, + }, + } + s.Name = cloudformation.Ref(name) + project.Secrets[i] = s + } + logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) template.Resources["LogGroup"] = &logs.LogGroup{ LogGroupName: logGroup, diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/pkg/amazon/backend/compatibility.go index 93806e96..ebf91588 100644 --- a/ecs/pkg/amazon/backend/compatibility.go +++ b/ecs/pkg/amazon/backend/compatibility.go @@ -37,8 +37,14 @@ var compatibleComposeAttributes = []string{ "services.ports.mode", "services.ports.target", "services.ports.protocol", + "services.secrets", + "services.secrets.source", + "services.secrets.target", "services.user", "services.working_dir", + "secrets.external", + "secrets.name", + "secrets.file", } func (c *FargateCompatibilityChecker) CheckImage(service *types.ServiceConfig) { diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index b4ae15ae..e024bde2 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -17,6 +17,8 @@ import ( "github.com/docker/ecs-plugin/pkg/compose" ) +const secretsInitContainerImage = "docker/ecs-secrets-sidecar" + func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { cpu, mem, err := toLimits(service) if err != nil { @@ -37,50 +39,118 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi fmt.Sprintf(" %s.local", project.Name), })) - return &ecs.TaskDefinition{ - ContainerDefinitions: []ecs.TaskDefinition_ContainerDefinition{ - { - Command: service.Command, - DisableNetworking: service.NetworkMode == "none", - DnsSearchDomains: service.DNSSearch, - DnsServers: service.DNS, - DockerSecurityOptions: service.SecurityOpt, - EntryPoint: service.Entrypoint, - Environment: toKeyValuePair(service.Environment), - Essential: true, - ExtraHosts: toHostEntryPtr(service.ExtraHosts), - FirelensConfiguration: nil, - HealthCheck: toHealthCheck(service.HealthCheck), - Hostname: service.Hostname, - Image: service.Image, - Interactive: false, - Links: nil, - LinuxParameters: toLinuxParameters(service), - LogConfiguration: &ecs.TaskDefinition_LogConfiguration{ - LogDriver: ecsapi.LogDriverAwslogs, - Options: map[string]string{ - "awslogs-region": cloudformation.Ref("AWS::Region"), - "awslogs-group": cloudformation.Ref("LogGroup"), - "awslogs-stream-prefix": project.Name, - }, - }, - MemoryReservation: memReservation, - Name: service.Name, - PortMappings: toPortMappings(service.Ports), - Privileged: service.Privileged, - PseudoTerminal: service.Tty, - ReadonlyRootFilesystem: service.ReadOnly, - RepositoryCredentials: credential, - ResourceRequirements: nil, - StartTimeout: 0, - StopTimeout: durationToInt(service.StopGracePeriod), - SystemControls: toSystemControls(service.Sysctls), - Ulimits: toUlimits(service.Ulimits), - User: service.User, - VolumesFrom: nil, - WorkingDirectory: service.WorkingDir, - }, + logConfiguration := &ecs.TaskDefinition_LogConfiguration{ + LogDriver: ecsapi.LogDriverAwslogs, + Options: map[string]string{ + "awslogs-region": cloudformation.Ref("AWS::Region"), + "awslogs-group": cloudformation.Ref("LogGroup"), + "awslogs-stream-prefix": project.Name, }, + } + + var ( + containers []ecs.TaskDefinition_ContainerDefinition + volumes []ecs.TaskDefinition_Volume + mounts []ecs.TaskDefinition_MountPoint + initContainers []ecs.TaskDefinition_ContainerDependency + ) + if len(service.Secrets) > 0 { + volumes = append(volumes, ecs.TaskDefinition_Volume{ + Name: "secrets", + }) + mounts = append(mounts, ecs.TaskDefinition_MountPoint{ + ContainerPath: "/run/secrets/", + ReadOnly: true, + SourceVolume: "secrets", + }) + initContainers = append(initContainers, ecs.TaskDefinition_ContainerDependency{ + Condition: ecsapi.ContainerConditionSuccess, + ContainerName: "Secrets_InitContainer", + }) + + var ( + names []string + secrets []ecs.TaskDefinition_Secret + ) + for _, s := range service.Secrets { + secretConfig := project.Secrets[s.Source] + if s.Target == "" { + s.Target = s.Source + } + secrets = append(secrets, ecs.TaskDefinition_Secret{ + Name: s.Target, + ValueFrom: secretConfig.Name, + }) + name := s.Target + if ext, ok := secretConfig.Extensions[compose.ExtensionKeys]; ok { + var keys []string + if key, ok := ext.(string); ok { + keys = append(keys, key) + } else { + for _, k := range ext.([]interface{}) { + keys = append(keys, k.(string)) + } + } + name = fmt.Sprintf("%s:%s", s.Target, strings.Join(keys, ",")) + } + names = append(names, name) + } + containers = append(containers, ecs.TaskDefinition_ContainerDefinition{ + Name: fmt.Sprintf("%s_Secrets_InitContainer", normalizeResourceName(service.Name)), + Image: secretsInitContainerImage, + Command: names, + Essential: false, // FIXME this will be ignored, see https://github.com/awslabs/goformation/issues/61#issuecomment-625139607 + LogConfiguration: logConfiguration, + MountPoints: []ecs.TaskDefinition_MountPoint{ + { + ContainerPath: "/run/secrets/", + ReadOnly: false, + SourceVolume: "secrets", + }, + }, + Secrets: secrets, + }) + } + + containers = append(containers, ecs.TaskDefinition_ContainerDefinition{ + Command: service.Command, + DisableNetworking: service.NetworkMode == "none", + DependsOnProp: initContainers, + DnsSearchDomains: service.DNSSearch, + DnsServers: service.DNS, + DockerSecurityOptions: service.SecurityOpt, + EntryPoint: service.Entrypoint, + Environment: toKeyValuePair(service.Environment), + Essential: true, + ExtraHosts: toHostEntryPtr(service.ExtraHosts), + FirelensConfiguration: nil, + HealthCheck: toHealthCheck(service.HealthCheck), + Hostname: service.Hostname, + Image: service.Image, + Interactive: false, + Links: nil, + LinuxParameters: toLinuxParameters(service), + LogConfiguration: logConfiguration, + MemoryReservation: memReservation, + MountPoints: mounts, + Name: service.Name, + PortMappings: toPortMappings(service.Ports), + Privileged: service.Privileged, + PseudoTerminal: service.Tty, + ReadonlyRootFilesystem: service.ReadOnly, + RepositoryCredentials: credential, + ResourceRequirements: nil, + StartTimeout: 0, + StopTimeout: durationToInt(service.StopGracePeriod), + SystemControls: toSystemControls(service.Sysctls), + Ulimits: toUlimits(service.Ulimits), + User: service.User, + VolumesFrom: nil, + WorkingDirectory: service.WorkingDir, + }) + + return &ecs.TaskDefinition{ + ContainerDefinitions: containers, Cpu: cpu, Family: fmt.Sprintf("%s-%s", project.Name, service.Name), IpcMode: service.Ipc, @@ -90,6 +160,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi PlacementConstraints: toPlacementConstraints(service.Deploy), ProxyConfiguration: nil, RequiresCompatibilities: []string{ecsapi.LaunchTypeFargate}, + Volumes: volumes, }, nil } diff --git a/ecs/pkg/amazon/cloudformation/marshall.go b/ecs/pkg/amazon/cloudformation/marshall.go new file mode 100644 index 00000000..034bf809 --- /dev/null +++ b/ecs/pkg/amazon/cloudformation/marshall.go @@ -0,0 +1,45 @@ +package cloudformation + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/awslabs/goformation/v4/cloudformation" +) + +func Marshall(template *cloudformation.Template) ([]byte, error) { + raw, err := template.JSON() + if err != nil { + return nil, err + } + + var unmarshalled interface{} + if err := json.Unmarshal(raw, &unmarshalled); err != nil { + return nil, fmt.Errorf("invalid JSON: %s", err) + } + + if input, ok := unmarshalled.(map[string]interface{}); ok { + if resources, ok := input["Resources"]; ok { + for _, uresource := range resources.(map[string]interface{}) { + if resource, ok := uresource.(map[string]interface{}); ok { + if resource["Type"] == "AWS::ECS::TaskDefinition" { + properties := resource["Properties"].(map[string]interface{}) + for _, def := range properties["ContainerDefinitions"].([]interface{}) { + containerDefinition := def.(map[string]interface{}) + if strings.HasSuffix(containerDefinition["Name"].(string), "_InitContainer") { + containerDefinition["Essential"] = "false" + } + } + } + } + } + } + } + + raw, err = json.MarshalIndent(unmarshalled, "", " ") + if err != nil { + return nil, fmt.Errorf("invalid JSON: %s", err) + } + return raw, err +} diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/pkg/amazon/sdk/sdk.go index da40fa52..057e9fff 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/pkg/amazon/sdk/sdk.go @@ -6,6 +6,8 @@ import ( "strings" "time" + cloudformation2 "github.com/docker/ecs-plugin/pkg/amazon/cloudformation" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" @@ -164,7 +166,7 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) { func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template, parameters map[string]string) error { logrus.Debug("Create CloudFormation stack") - json, err := template.JSON() + json, err := cloudformation2.Marshall(template) if err != nil { return err } @@ -192,7 +194,7 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template func (s sdk) CreateChangeSet(ctx context.Context, name string, template *cf.Template, parameters map[string]string) (string, error) { logrus.Debug("Create CloudFormation Changeset") - json, err := template.JSON() + json, err := cloudformation2.Marshall(template) if err != nil { return "", err } diff --git a/ecs/pkg/compose/x.go b/ecs/pkg/compose/x.go index 8e52a368..777987af 100644 --- a/ecs/pkg/compose/x.go +++ b/ecs/pkg/compose/x.go @@ -6,4 +6,5 @@ const ( ExtensionPullCredentials = "x-aws-pull_credentials" ExtensionLB = "x-aws-loadbalancer" ExtensionCluster = "x-aws-cluster" + ExtensionKeys = "x-aws-keys" ) diff --git a/ecs/secrets/Dockerfile b/ecs/secrets/Dockerfile new file mode 100644 index 00000000..7395d125 --- /dev/null +++ b/ecs/secrets/Dockerfile @@ -0,0 +1,8 @@ +FROM golang:1.14.4-alpine AS builder +WORKDIR $GOPATH/src/github.com/docker/ecs-secrets +COPY . . +RUN GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /go/bin/secrets + +FROM scratch +COPY --from=builder /go/bin/secrets /secrets +ENTRYPOINT ["/secrets"] diff --git a/ecs/secrets/main.go b/ecs/secrets/main.go new file mode 100644 index 00000000..cc37325a --- /dev/null +++ b/ecs/secrets/main.go @@ -0,0 +1,85 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +// return codes: +// 1: failed to read secret from env +// 2: failed to parse hierarchical secret +// 3: failed to write secret content into file +func main() { + for _, name := range os.Args[1:] { + i := strings.Index(name, ":") + var keys []string + if i > 0 { + keys = strings.Split(name[i+1:], ",") + name = name[:i] + } + value, ok := os.LookupEnv(name) + if !ok { + fmt.Fprintf(os.Stderr, "%q variable not set", name) + os.Exit(1) + } + + secrets := filepath.Join("/run/secrets", name) + + if len(keys) == 0 { + // raw secret + fmt.Printf("inject secret %q info %s\n", name, secrets) + err := ioutil.WriteFile(secrets, []byte(value), 0444) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(3) + } + os.Exit(0) + } + + var unmarshalled interface{} + err := json.Unmarshal([]byte(value), &unmarshalled) + if err == nil { + if dict, ok := unmarshalled.(map[string]interface{}); ok { + os.MkdirAll(secrets, 0555) + for k, v := range dict { + if !contains(keys, k) && !contains(keys, "*") { + continue + } + path := filepath.Join(secrets, k) + fmt.Printf("inject secret %q info %s\n", k, path) + + var raw []byte + if s, ok := v.(string); ok { + raw = []byte(s) + } else { + raw, err = json.Marshal(v) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(2) + } + } + + err = ioutil.WriteFile(path, raw, 0444) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(3) + } + } + os.Exit(0) + } + } + } +} + +func contains(keys []string, s string) bool { + for _, k := range keys { + if k == s { + return true + } + } + return false +} From 4bfab35007abee926201cb4791546f245bb194c4 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 10 Aug 2020 16:15:46 +0200 Subject: [PATCH 188/205] TestCase for the secrets init container Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 9 +- ecs/go.mod | 1 + ecs/pkg/amazon/backend/cloudformation.go | 3 +- ecs/secrets/main.go | 149 ++++++++++++++--------- ecs/secrets/main_test.go | 105 ++++++++++++++++ 5 files changed, 203 insertions(+), 64 deletions(-) create mode 100644 ecs/secrets/main_test.go diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index f4e6d4e9..d0f1a224 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -7,11 +7,10 @@ import ( "os" "strings" - "github.com/docker/ecs-plugin/pkg/amazon/cloudformation" - "github.com/compose-spec/compose-go/cli" "github.com/docker/cli/cli/command" amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" + "github.com/docker/ecs-plugin/pkg/amazon/cloudformation" "github.com/docker/ecs-plugin/pkg/docker" "github.com/spf13/cobra" ) @@ -60,13 +59,11 @@ func ConvertCommand(dockerCli command.Cli, options *composeOptions) *cobra.Comma if err != nil { return err } - json, err := cloudformation.Marshall(template) if err != nil { - fmt.Printf("Failed to generate JSON: %s\n", err) - } else { - fmt.Printf("%s\n", string(json)) + return err } + fmt.Printf("%s\n", string(json)) return nil }), } diff --git a/ecs/go.mod b/ecs/go.mod index f3e2949d..c7f1ad67 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -39,6 +39,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/onsi/ginkgo v1.11.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.6.0 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.5 diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index a47e1ad6..e5410858 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -6,8 +6,6 @@ import ( "regexp" "strings" - "github.com/awslabs/goformation/v4/cloudformation/secretsmanager" - ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/elbv2" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" @@ -17,6 +15,7 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/awslabs/goformation/v4/cloudformation/iam" "github.com/awslabs/goformation/v4/cloudformation/logs" + "github.com/awslabs/goformation/v4/cloudformation/secretsmanager" cloudmap "github.com/awslabs/goformation/v4/cloudformation/servicediscovery" "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/compose-spec/compose-go/compatibility" diff --git a/ecs/secrets/main.go b/ecs/secrets/main.go index cc37325a..eeaa992e 100644 --- a/ecs/secrets/main.go +++ b/ecs/secrets/main.go @@ -7,72 +7,109 @@ import ( "os" "path/filepath" "strings" + + "github.com/pkg/errors" ) -// return codes: -// 1: failed to read secret from env -// 2: failed to parse hierarchical secret -// 3: failed to write secret content into file +type secret struct { + name string + keys []string +} + +const secretsFolder = "/run/secrets" + func main() { - for _, name := range os.Args[1:] { + secrets := parseInput(os.Args[1:]) + + for _, secret := range secrets { + err := createSecretFiles(secret, secretsFolder) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + } +} + +func createSecretFiles(secret secret, path string) error { + value, ok := os.LookupEnv(secret.name) + if !ok { + return fmt.Errorf("%q variable not set", secret.name) + } + + secrets := filepath.Join(path, secret.name) + + if len(secret.keys) == 0 { + // raw secret + fmt.Printf("inject secret %q info %s\n", secret.name, secrets) + return ioutil.WriteFile(secrets, []byte(value), 0444) + } + + var unmarshalled interface{} + err := json.Unmarshal([]byte(value), &unmarshalled) + if err != nil { + return errors.Wrapf(err, "%q secret is not a valid JSON document", secret.name) + } + + dict, ok := unmarshalled.(map[string]interface{}) + if !ok { + return errors.Wrapf(err, "%q secret is not a JSON dictionary", secret.name) + } + err = os.MkdirAll(secrets, 0755) + if err != nil { + return err + } + + if contains(secret.keys, "*") { + var keys []string + for k := range dict { + keys = append(keys, k) + } + secret.keys = keys + } + + for _, k := range secret.keys { + path := filepath.Join(secrets, k) + fmt.Printf("inject secret %q info %s\n", k, path) + + v, ok := dict[k] + if !ok { + return fmt.Errorf("%q secret has no %q key", secret.name, k) + } + + var raw []byte + if s, ok := v.(string); ok { + raw = []byte(s) + } else { + raw, err = json.Marshal(v) + if err != nil { + return err + } + } + + err = ioutil.WriteFile(path, raw, 0444) + if err != nil { + return err + } + } + return nil +} + +// parseInput parse secret to be dumped into secret files with syntax `VARIABLE_NAME[:COMA_SEPARATED_KEYS]` +func parseInput(input []string) []secret { + var secrets []secret + for _, name := range input { i := strings.Index(name, ":") var keys []string if i > 0 { keys = strings.Split(name[i+1:], ",") name = name[:i] } - value, ok := os.LookupEnv(name) - if !ok { - fmt.Fprintf(os.Stderr, "%q variable not set", name) - os.Exit(1) - } - - secrets := filepath.Join("/run/secrets", name) - - if len(keys) == 0 { - // raw secret - fmt.Printf("inject secret %q info %s\n", name, secrets) - err := ioutil.WriteFile(secrets, []byte(value), 0444) - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(3) - } - os.Exit(0) - } - - var unmarshalled interface{} - err := json.Unmarshal([]byte(value), &unmarshalled) - if err == nil { - if dict, ok := unmarshalled.(map[string]interface{}); ok { - os.MkdirAll(secrets, 0555) - for k, v := range dict { - if !contains(keys, k) && !contains(keys, "*") { - continue - } - path := filepath.Join(secrets, k) - fmt.Printf("inject secret %q info %s\n", k, path) - - var raw []byte - if s, ok := v.(string); ok { - raw = []byte(s) - } else { - raw, err = json.Marshal(v) - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(2) - } - } - - err = ioutil.WriteFile(path, raw, 0444) - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(3) - } - } - os.Exit(0) - } - } + secrets = append(secrets, secret{ + name: name, + keys: keys, + }) } + return secrets } func contains(keys []string, s string) bool { diff --git a/ecs/secrets/main_test.go b/ecs/secrets/main_test.go new file mode 100644 index 00000000..ff6f630f --- /dev/null +++ b/ecs/secrets/main_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" +) + +func TestParseSecrets(t *testing.T) { + secrets := parseInput([]string{ + "foo", + "bar:*", + "zot:key0,key1", + }) + assert.Check(t, len(secrets) == 3) + assert.Check(t, secrets[0].name == "foo") + assert.Check(t, secrets[0].keys == nil) + + assert.Check(t, secrets[1].name == "bar") + assert.Check(t, len(secrets[1].keys) == 1) + assert.Check(t, secrets[1].keys[0] == "*") + + assert.Check(t, secrets[2].name == "zot") + assert.Check(t, len(secrets[2].keys) == 2) + assert.Check(t, secrets[2].keys[0] == "key0") + assert.Check(t, secrets[2].keys[1] == "key1") +} + +func TestRawSecret(t *testing.T) { + dir := fs.NewDir(t, "secrets").Path() + os.Setenv("raw", "something_secret") + defer os.Unsetenv("raw") + + err := createSecretFiles(secret{ + name: "raw", + keys: nil, + }, dir) + assert.NilError(t, err) + file, err := ioutil.ReadFile(filepath.Join(dir, "raw")) + assert.NilError(t, err) + content := string(file) + assert.Equal(t, content, "something_secret") +} + +func TestSelectedKeysSecret(t *testing.T) { + dir := fs.NewDir(t, "secrets").Path() + os.Setenv("json", ` +{ + "foo": "bar", + "zot": "qix" +}`) + defer os.Unsetenv("json") + + err := createSecretFiles(secret{ + name: "json", + keys: []string{"foo"}, + }, dir) + assert.NilError(t, err) + file, err := ioutil.ReadFile(filepath.Join(dir, "json", "foo")) + assert.NilError(t, err) + content := string(file) + assert.Equal(t, content, "bar") + + _, err = os.Stat(filepath.Join(dir, "json", "zot")) + assert.Check(t, os.IsNotExist(err)) +} + +func TestAllKeysSecret(t *testing.T) { + dir := fs.NewDir(t, "secrets").Path() + os.Setenv("json", ` +{ + "foo": "bar", + "zot": "qix" +}`) + defer os.Unsetenv("json") + + err := createSecretFiles(secret{ + name: "json", + keys: []string{"*"}, + }, dir) + assert.NilError(t, err) + file, err := ioutil.ReadFile(filepath.Join(dir, "json", "foo")) + assert.NilError(t, err) + content := string(file) + assert.Equal(t, content, "bar") + + file, err = ioutil.ReadFile(filepath.Join(dir, "json", "zot")) + assert.NilError(t, err) + content = string(file) + assert.Equal(t, content, "qix") +} + +func TestUnknownSecret(t *testing.T) { + dir := fs.NewDir(t, "secrets").Path() + + err := createSecretFiles(secret{ + name: "not_set", + keys: nil, + }, dir) + assert.Check(t, err != nil) +} From d74796aca22d7ec9d97a620576f6c133998f9ead Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 10 Aug 2020 19:02:27 +0200 Subject: [PATCH 189/205] Pass secret definition to init container as json struct this avoid yet another new micro-formats that is poorly documented Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/convert.go | 31 ++++-- ecs/secrets/Dockerfile | 4 +- ecs/secrets/init.go | 87 +++++++++++++++ ecs/secrets/{main_test.go => init_test.go} | 46 +++----- ecs/secrets/main.go | 122 --------------------- ecs/secrets/main/main.go | 33 ++++++ 6 files changed, 155 insertions(+), 168 deletions(-) create mode 100644 ecs/secrets/init.go rename ecs/secrets/{main_test.go => init_test.go} (64%) delete mode 100644 ecs/secrets/main.go create mode 100644 ecs/secrets/main/main.go diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index e024bde2..3eddde17 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -1,12 +1,15 @@ package backend import ( + "encoding/json" "fmt" "sort" "strconv" "strings" "time" + "github.com/docker/ecs-plugin/secrets" + "github.com/aws/aws-sdk-go/aws" ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" @@ -55,6 +58,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi initContainers []ecs.TaskDefinition_ContainerDependency ) if len(service.Secrets) > 0 { + initContainerName := fmt.Sprintf("%s_Secrets_InitContainer", normalizeResourceName(service.Name)) volumes = append(volumes, ecs.TaskDefinition_Volume{ Name: "secrets", }) @@ -65,25 +69,24 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi }) initContainers = append(initContainers, ecs.TaskDefinition_ContainerDependency{ Condition: ecsapi.ContainerConditionSuccess, - ContainerName: "Secrets_InitContainer", + ContainerName: initContainerName, }) var ( - names []string - secrets []ecs.TaskDefinition_Secret + args []secrets.Secret + taskSecrets []ecs.TaskDefinition_Secret ) for _, s := range service.Secrets { secretConfig := project.Secrets[s.Source] if s.Target == "" { s.Target = s.Source } - secrets = append(secrets, ecs.TaskDefinition_Secret{ + taskSecrets = append(taskSecrets, ecs.TaskDefinition_Secret{ Name: s.Target, ValueFrom: secretConfig.Name, }) - name := s.Target + var keys []string if ext, ok := secretConfig.Extensions[compose.ExtensionKeys]; ok { - var keys []string if key, ok := ext.(string); ok { keys = append(keys, key) } else { @@ -91,14 +94,20 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi keys = append(keys, k.(string)) } } - name = fmt.Sprintf("%s:%s", s.Target, strings.Join(keys, ",")) } - names = append(names, name) + args = append(args, secrets.Secret{ + Name: s.Target, + Keys: keys, + }) + } + command, err := json.Marshal(args) + if err != nil { + return nil, err } containers = append(containers, ecs.TaskDefinition_ContainerDefinition{ - Name: fmt.Sprintf("%s_Secrets_InitContainer", normalizeResourceName(service.Name)), + Name: initContainerName, Image: secretsInitContainerImage, - Command: names, + Command: []string{string(command)}, Essential: false, // FIXME this will be ignored, see https://github.com/awslabs/goformation/issues/61#issuecomment-625139607 LogConfiguration: logConfiguration, MountPoints: []ecs.TaskDefinition_MountPoint{ @@ -108,7 +117,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi SourceVolume: "secrets", }, }, - Secrets: secrets, + Secrets: taskSecrets, }) } diff --git a/ecs/secrets/Dockerfile b/ecs/secrets/Dockerfile index 7395d125..638e113b 100644 --- a/ecs/secrets/Dockerfile +++ b/ecs/secrets/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.14.4-alpine AS builder -WORKDIR $GOPATH/src/github.com/docker/ecs-secrets +WORKDIR $GOPATH/src/github.com/docker/ecs-plugin/secrets COPY . . -RUN GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /go/bin/secrets +RUN GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /go/bin/secrets main/main.go FROM scratch COPY --from=builder /go/bin/secrets /secrets diff --git a/ecs/secrets/init.go b/ecs/secrets/init.go new file mode 100644 index 00000000..1903d3cc --- /dev/null +++ b/ecs/secrets/init.go @@ -0,0 +1,87 @@ +package secrets + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +type Secret struct { + Name string + Keys []string +} + +func CreateSecretFiles(secret Secret, path string) error { + value, ok := os.LookupEnv(secret.Name) + if !ok { + return fmt.Errorf("%q variable not set", secret.Name) + } + + secrets := filepath.Join(path, secret.Name) + + if len(secret.Keys) == 0 { + // raw Secret + fmt.Printf("inject Secret %q info %s\n", secret.Name, secrets) + return ioutil.WriteFile(secrets, []byte(value), 0444) + } + + var unmarshalled interface{} + err := json.Unmarshal([]byte(value), &unmarshalled) + if err != nil { + return fmt.Errorf("%q Secret is not a valid JSON document: %w", secret.Name, err) + } + + dict, ok := unmarshalled.(map[string]interface{}) + if !ok { + return fmt.Errorf("%q Secret is not a JSON dictionary: %w", secret.Name, err) + } + err = os.MkdirAll(secrets, 0755) + if err != nil { + return err + } + + if contains(secret.Keys, "*") { + var keys []string + for k := range dict { + keys = append(keys, k) + } + secret.Keys = keys + } + + for _, k := range secret.Keys { + path := filepath.Join(secrets, k) + fmt.Printf("inject Secret %q info %s\n", k, path) + + v, ok := dict[k] + if !ok { + return fmt.Errorf("%q Secret has no %q key", secret.Name, k) + } + + var raw []byte + if s, ok := v.(string); ok { + raw = []byte(s) + } else { + raw, err = json.Marshal(v) + if err != nil { + return err + } + } + + err = ioutil.WriteFile(path, raw, 0444) + if err != nil { + return err + } + } + return nil +} + +func contains(keys []string, s string) bool { + for _, k := range keys { + if k == s { + return true + } + } + return false +} diff --git a/ecs/secrets/main_test.go b/ecs/secrets/init_test.go similarity index 64% rename from ecs/secrets/main_test.go rename to ecs/secrets/init_test.go index ff6f630f..dd068f5a 100644 --- a/ecs/secrets/main_test.go +++ b/ecs/secrets/init_test.go @@ -1,4 +1,4 @@ -package main +package secrets import ( "io/ioutil" @@ -10,34 +10,14 @@ import ( "gotest.tools/v3/fs" ) -func TestParseSecrets(t *testing.T) { - secrets := parseInput([]string{ - "foo", - "bar:*", - "zot:key0,key1", - }) - assert.Check(t, len(secrets) == 3) - assert.Check(t, secrets[0].name == "foo") - assert.Check(t, secrets[0].keys == nil) - - assert.Check(t, secrets[1].name == "bar") - assert.Check(t, len(secrets[1].keys) == 1) - assert.Check(t, secrets[1].keys[0] == "*") - - assert.Check(t, secrets[2].name == "zot") - assert.Check(t, len(secrets[2].keys) == 2) - assert.Check(t, secrets[2].keys[0] == "key0") - assert.Check(t, secrets[2].keys[1] == "key1") -} - func TestRawSecret(t *testing.T) { dir := fs.NewDir(t, "secrets").Path() os.Setenv("raw", "something_secret") defer os.Unsetenv("raw") - err := createSecretFiles(secret{ - name: "raw", - keys: nil, + err := CreateSecretFiles(Secret{ + Name: "raw", + Keys: nil, }, dir) assert.NilError(t, err) file, err := ioutil.ReadFile(filepath.Join(dir, "raw")) @@ -55,9 +35,9 @@ func TestSelectedKeysSecret(t *testing.T) { }`) defer os.Unsetenv("json") - err := createSecretFiles(secret{ - name: "json", - keys: []string{"foo"}, + err := CreateSecretFiles(Secret{ + Name: "json", + Keys: []string{"foo"}, }, dir) assert.NilError(t, err) file, err := ioutil.ReadFile(filepath.Join(dir, "json", "foo")) @@ -78,9 +58,9 @@ func TestAllKeysSecret(t *testing.T) { }`) defer os.Unsetenv("json") - err := createSecretFiles(secret{ - name: "json", - keys: []string{"*"}, + err := CreateSecretFiles(Secret{ + Name: "json", + Keys: []string{"*"}, }, dir) assert.NilError(t, err) file, err := ioutil.ReadFile(filepath.Join(dir, "json", "foo")) @@ -97,9 +77,9 @@ func TestAllKeysSecret(t *testing.T) { func TestUnknownSecret(t *testing.T) { dir := fs.NewDir(t, "secrets").Path() - err := createSecretFiles(secret{ - name: "not_set", - keys: nil, + err := CreateSecretFiles(Secret{ + Name: "not_set", + Keys: nil, }, dir) assert.Check(t, err != nil) } diff --git a/ecs/secrets/main.go b/ecs/secrets/main.go deleted file mode 100644 index eeaa992e..00000000 --- a/ecs/secrets/main.go +++ /dev/null @@ -1,122 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" -) - -type secret struct { - name string - keys []string -} - -const secretsFolder = "/run/secrets" - -func main() { - secrets := parseInput(os.Args[1:]) - - for _, secret := range secrets { - err := createSecretFiles(secret, secretsFolder) - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(1) - } - } -} - -func createSecretFiles(secret secret, path string) error { - value, ok := os.LookupEnv(secret.name) - if !ok { - return fmt.Errorf("%q variable not set", secret.name) - } - - secrets := filepath.Join(path, secret.name) - - if len(secret.keys) == 0 { - // raw secret - fmt.Printf("inject secret %q info %s\n", secret.name, secrets) - return ioutil.WriteFile(secrets, []byte(value), 0444) - } - - var unmarshalled interface{} - err := json.Unmarshal([]byte(value), &unmarshalled) - if err != nil { - return errors.Wrapf(err, "%q secret is not a valid JSON document", secret.name) - } - - dict, ok := unmarshalled.(map[string]interface{}) - if !ok { - return errors.Wrapf(err, "%q secret is not a JSON dictionary", secret.name) - } - err = os.MkdirAll(secrets, 0755) - if err != nil { - return err - } - - if contains(secret.keys, "*") { - var keys []string - for k := range dict { - keys = append(keys, k) - } - secret.keys = keys - } - - for _, k := range secret.keys { - path := filepath.Join(secrets, k) - fmt.Printf("inject secret %q info %s\n", k, path) - - v, ok := dict[k] - if !ok { - return fmt.Errorf("%q secret has no %q key", secret.name, k) - } - - var raw []byte - if s, ok := v.(string); ok { - raw = []byte(s) - } else { - raw, err = json.Marshal(v) - if err != nil { - return err - } - } - - err = ioutil.WriteFile(path, raw, 0444) - if err != nil { - return err - } - } - return nil -} - -// parseInput parse secret to be dumped into secret files with syntax `VARIABLE_NAME[:COMA_SEPARATED_KEYS]` -func parseInput(input []string) []secret { - var secrets []secret - for _, name := range input { - i := strings.Index(name, ":") - var keys []string - if i > 0 { - keys = strings.Split(name[i+1:], ",") - name = name[:i] - } - secrets = append(secrets, secret{ - name: name, - keys: keys, - }) - } - return secrets -} - -func contains(keys []string, s string) bool { - for _, k := range keys { - if k == s { - return true - } - } - return false -} diff --git a/ecs/secrets/main/main.go b/ecs/secrets/main/main.go new file mode 100644 index 00000000..703f88f8 --- /dev/null +++ b/ecs/secrets/main/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/docker/ecs-plugin/secrets" +) + +const secretsFolder = "/run/secrets" + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "usage: secrets ") + os.Exit(1) + } + + var input []secrets.Secret + err := json.Unmarshal([]byte(os.Args[1]), &input) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + for _, secret := range input { + err := secrets.CreateSecretFiles(secret, secretsFolder) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + } +} From 8e538683d33a61ce06a7deb1a9486f79137bffda Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 11 Aug 2020 08:37:48 +0200 Subject: [PATCH 190/205] Update Golden Signed-off-by: Nicolas De Loof --- .../testdata/simple/simple-cloudformation-conversion.golden | 1 - 1 file changed, 1 deletion(-) diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden index c61fc649..d42a2464 100644 --- a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden @@ -207,7 +207,6 @@ }, "SimpleTCP80TargetGroup": { "Properties": { - "Name": "SimpleTCP80TargetGroup", "Port": 80, "Protocol": "HTTP", "Tags": [ From 7d927ebe4f0aeed964a788d53950c5e2a24b7681 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 11 Aug 2020 11:19:58 +0200 Subject: [PATCH 191/205] Compute rolling update min/max limits Signed-off-by: Nicolas De Loof --- ecs/go.mod | 6 +-- ecs/go.sum | 22 +++++++++ ecs/pkg/amazon/backend/cloudformation.go | 49 ++++++++++++++++++- ecs/pkg/amazon/backend/cloudformation_test.go | 41 +++++++++++----- ecs/pkg/amazon/backend/compatibility.go | 2 + ecs/pkg/compose/x.go | 2 + 6 files changed, 105 insertions(+), 17 deletions(-) diff --git a/ecs/go.mod b/ecs/go.mod index c7f1ad67..06637206 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -14,7 +14,7 @@ require ( github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3 + github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457 github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 @@ -28,14 +28,13 @@ require ( github.com/gogo/protobuf v1.3.1 // indirect github.com/gorilla/mux v1.7.3 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/imdario/mergo v0.3.10 // indirect github.com/jinzhu/gorm v1.9.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lib/pq v1.3.0 // indirect github.com/manifoldco/promptui v0.7.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/miekg/pkcs11 v1.0.3 // indirect - github.com/mitchellh/mapstructure v1.3.2 + github.com/mitchellh/mapstructure v1.3.3 github.com/morikuni/aec v1.0.0 // indirect github.com/onsi/ginkgo v1.11.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect @@ -46,7 +45,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/theupdateframework/notary v0.6.1 // indirect github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect - golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/grpc v1.27.0 // indirect gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect diff --git a/ecs/go.sum b/ecs/go.sum index 96406699..d73c6e3d 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -56,6 +56,8 @@ github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:ea github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3 h1:+ntlMTrEcScJjlnEOP8P1IIrusJaR93Eazr66YgUueA= github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= +github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457 h1:8ely1LF7H02sIWz6QjgU53YBCiRpYlM9F9u1MeE1ZPk= +github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457/go.mod h1:cS0vAvM6u9yjJgKWIH2yiqYMWO7WGJb+c0Irw+RefqU= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -137,6 +139,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -227,6 +231,9 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mjibson/esc v0.2.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -344,6 +351,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= @@ -359,13 +367,17 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90Pveol golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -378,15 +390,19 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYR golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -405,6 +421,8 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -420,6 +438,10 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index e5410858..482cf121 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -190,6 +190,11 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro dependsOn = append(dependsOn, serviceResourceName(dependency)) } + minPercent, maxPercent, err := b.computeRollingUpdateLimits(service) + if err != nil { + return nil, err + } + template.Resources[serviceResourceName(service.Name)] = &ecs.Service{ AWSCloudFormationDependsOn: dependsOn, Cluster: cluster, @@ -198,8 +203,8 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro Type: ecsapi.DeploymentControllerTypeEcs, }, DeploymentConfiguration: &ecs.Service_DeploymentConfiguration{ - MaximumPercent: 200, - MinimumHealthyPercent: 100, + MaximumPercent: maxPercent, + MinimumHealthyPercent: minPercent, }, LaunchType: ecsapi.LaunchTypeFargate, LoadBalancers: serviceLB, @@ -232,6 +237,46 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro return template, nil } +func (b Backend) computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) { + maxPercent := 200 + minPercent := 100 + if service.Deploy == nil || service.Deploy.UpdateConfig == nil { + return minPercent, maxPercent, nil + } + updateConfig := service.Deploy.UpdateConfig + min, okMin := updateConfig.Extensions[compose.ExtensionMinPercent] + if okMin { + minPercent = min.(int) + } + max, okMax := updateConfig.Extensions[compose.ExtensionMaxPercent] + if okMax { + maxPercent = max.(int) + } + if okMin && okMax { + return minPercent, maxPercent, nil + } + + if updateConfig.Parallelism != nil { + parallelism := int(*updateConfig.Parallelism) + if service.Deploy.Replicas == nil { + return minPercent, maxPercent, + fmt.Errorf("rolling update configuration require deploy.replicas to be set") + } + replicas := int(*service.Deploy.Replicas) + if replicas < parallelism { + return minPercent, maxPercent, + fmt.Errorf("deploy.replicas (%d) must be greater than deploy.update_config.parallelism (%d)", replicas, parallelism) + } + if !okMin { + minPercent = (replicas - parallelism) * 100 / replicas + } + if !okMax { + maxPercent = (replicas + parallelism) * 100 / replicas + } + } + return minPercent, maxPercent, nil +} + func getLoadBalancerType(project *types.Project) string { for _, service := range project.Services { for _, port := range service.Ports { diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 0c7edd8f..880de0d6 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -26,9 +26,38 @@ func TestSimpleConvert(t *testing.T) { golden.Assert(t, result, expected) } +func TestRollingUpdateLimits(t *testing.T) { + template := convertYaml(t, "test", ` +services: + foo: + image: hello_world + deploy: + replicas: 4 + update_config: + parallelism: 2 +`) + service := template.Resources["FooService"].(*ecs.Service) + assert.Check(t, service.DeploymentConfiguration.MaximumPercent == 150) + assert.Check(t, service.DeploymentConfiguration.MinimumHealthyPercent == 50) +} + +func TestRollingUpdateExtension(t *testing.T) { + template := convertYaml(t, "test", ` +services: + foo: + image: hello_world + deploy: + update_config: + x-aws-min_percent: 25 + x-aws-max_percent: 125 +`) + service := template.Resources["FooService"].(*ecs.Service) + assert.Check(t, service.DeploymentConfiguration.MaximumPercent == 125) + assert.Check(t, service.DeploymentConfiguration.MinimumHealthyPercent == 25) +} + func TestRolePolicy(t *testing.T) { template := convertYaml(t, "test", ` -version: "3" services: foo: image: hello_world @@ -48,7 +77,6 @@ services: func TestMapNetworksToSecurityGroups(t *testing.T) { template := convertYaml(t, "test", ` -version: "3" services: test: image: hello_world @@ -73,7 +101,6 @@ networks: func TestLoadBalancerTypeApplication(t *testing.T) { template := convertYaml(t, "test123456789009876543211234567890", ` -version: "3" services: test: image: nginx @@ -89,7 +116,6 @@ services: func TestNoLoadBalancerIfNoPortExposed(t *testing.T) { template := convertYaml(t, "test", ` -version: "3" services: test: image: nginx @@ -105,7 +131,6 @@ services: func TestServiceReplicas(t *testing.T) { template := convertYaml(t, "test", ` -version: "3" services: test: image: nginx @@ -119,7 +144,6 @@ services: func TestTaskSizeConvert(t *testing.T) { template := convertYaml(t, "test", ` -version: "3" services: test: image: nginx @@ -137,7 +161,6 @@ services: assert.Equal(t, def.Memory, "2048") template = convertYaml(t, "test", ` -version: "3" services: test: image: nginx @@ -156,7 +179,6 @@ services: } func TestTaskSizeConvertFailure(t *testing.T) { model := loadConfig(t, "test", ` -version: "3" services: test: image: nginx @@ -172,7 +194,6 @@ services: func TestLoadBalancerTypeNetwork(t *testing.T) { template := convertYaml(t, "test", ` -version: "3" services: test: image: nginx @@ -187,7 +208,6 @@ services: func TestServiceMapping(t *testing.T) { template := convertYaml(t, "test", ` -version: "3" services: test: image: "image" @@ -227,7 +247,6 @@ func get(l []ecs.TaskDefinition_KeyValuePair, name string) string { func TestResourcesHaveProjectTagSet(t *testing.T) { template := convertYaml(t, "test", ` -version: "3" services: test: image: nginx diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/pkg/amazon/backend/compatibility.go index ebf91588..6857c498 100644 --- a/ecs/pkg/amazon/backend/compatibility.go +++ b/ecs/pkg/amazon/backend/compatibility.go @@ -22,6 +22,8 @@ var compatibleComposeAttributes = []string{ "services.deploy.resources.reservations", "services.deploy.resources.reservations.cpus", "services.deploy.resources.reservations.memory", + "services.deploy.update_config", + "services.deploy.update_config.parallelism", "services.entrypoint", "services.environment", "service.image", diff --git a/ecs/pkg/compose/x.go b/ecs/pkg/compose/x.go index 777987af..44ebfae9 100644 --- a/ecs/pkg/compose/x.go +++ b/ecs/pkg/compose/x.go @@ -7,4 +7,6 @@ const ( ExtensionLB = "x-aws-loadbalancer" ExtensionCluster = "x-aws-cluster" ExtensionKeys = "x-aws-keys" + ExtensionMinPercent = "x-aws-min_percent" + ExtensionMaxPercent = "x-aws-max_percent" ) From 5ed328d8df46cf2bc7ebd47369963135d48b57a0 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 11 Aug 2020 15:48:12 +0200 Subject: [PATCH 192/205] Add service.env_file support Signed-off-by: Nicolas De Loof --- ecs/go.mod | 1 + ecs/pkg/amazon/backend/cloudformation_test.go | 30 ++++++++ ecs/pkg/amazon/backend/compatibility.go | 1 + ecs/pkg/amazon/backend/convert.go | 74 +++++++++++++------ ecs/pkg/amazon/backend/testdata/input/envfile | 1 + 5 files changed, 85 insertions(+), 22 deletions(-) create mode 100644 ecs/pkg/amazon/backend/testdata/input/envfile diff --git a/ecs/go.mod b/ecs/go.mod index 06637206..594aa539 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -29,6 +29,7 @@ require ( github.com/gorilla/mux v1.7.3 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/jinzhu/gorm v1.9.12 // indirect + github.com/joho/godotenv v1.3.0 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lib/pq v1.3.0 // indirect github.com/manifoldco/promptui v0.7.0 diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 880de0d6..84ddf530 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -26,6 +26,36 @@ func TestSimpleConvert(t *testing.T) { golden.Assert(t, result, expected) } +func TestEnvFile(t *testing.T) { + template := convertYaml(t, "test", ` +services: + foo: + image: hello_world + env_file: + - testdata/input/envfile +`) + def := template.Resources["FooTaskDefinition"].(*ecs.TaskDefinition) + env := def.ContainerDefinitions[0].Environment + assert.Equal(t, env[0].Name, "FOO") + assert.Equal(t, env[0].Value, "BAR") +} + +func TestEnvFileAndEnv(t *testing.T) { + template := convertYaml(t, "test", ` +services: + foo: + image: hello_world + env_file: + - testdata/input/envfile + environment: + - "FOO=ZOT" +`) + def := template.Resources["FooTaskDefinition"].(*ecs.TaskDefinition) + env := def.ContainerDefinitions[0].Environment + assert.Equal(t, env[0].Name, "FOO") + assert.Equal(t, env[0].Value, "ZOT") +} + func TestRollingUpdateLimits(t *testing.T) { template := convertYaml(t, "test", ` services: diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/pkg/amazon/backend/compatibility.go index 6857c498..02ef5e50 100644 --- a/ecs/pkg/amazon/backend/compatibility.go +++ b/ecs/pkg/amazon/backend/compatibility.go @@ -26,6 +26,7 @@ var compatibleComposeAttributes = []string{ "services.deploy.update_config.parallelism", "services.entrypoint", "services.environment", + "services.env_file", "service.image", "services.init", "services.healthcheck", diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index 3eddde17..b130d51a 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -3,13 +3,13 @@ package backend import ( "encoding/json" "fmt" + "os" + "path/filepath" "sort" "strconv" "strings" "time" - "github.com/docker/ecs-plugin/secrets" - "github.com/aws/aws-sdk-go/aws" ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" @@ -18,6 +18,8 @@ import ( "github.com/compose-spec/compose-go/types" "github.com/docker/cli/opts" "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/secrets" + "github.com/joho/godotenv" ) const secretsInitContainerImage = "docker/ecs-secrets-sidecar" @@ -121,6 +123,11 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi }) } + pairs, err := createEnvironment(project, service) + if err != nil { + return nil, err + } + containers = append(containers, ecs.TaskDefinition_ContainerDefinition{ Command: service.Command, DisableNetworking: service.NetworkMode == "none", @@ -129,7 +136,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi DnsServers: service.DNS, DockerSecurityOptions: service.SecurityOpt, EntryPoint: service.Entrypoint, - Environment: toKeyValuePair(service.Environment), + Environment: pairs, Essential: true, ExtraHosts: toHostEntryPtr(service.ExtraHosts), FirelensConfiguration: nil, @@ -173,6 +180,48 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi }, nil } +func createEnvironment(project *types.Project, service types.ServiceConfig) ([]ecs.TaskDefinition_KeyValuePair, error) { + environment := map[string]*string{} + for _, f := range service.EnvFile { + if !filepath.IsAbs(f) { + f = filepath.Join(project.WorkingDir, f) + } + if _, err := os.Stat(f); os.IsNotExist(err) { + return nil, err + } + file, err := os.Open(f) + if err != nil { + return nil, err + } + defer file.Close() + + env, err := godotenv.Parse(file) + if err != nil { + return nil, err + } + for k, v := range env { + environment[k] = &v + } + } + for k, v := range service.Environment { + environment[k] = v + } + + var pairs []ecs.TaskDefinition_KeyValuePair + for k, v := range environment { + name := k + var value string + if v != nil { + value = *v + } + pairs = append(pairs, ecs.TaskDefinition_KeyValuePair{ + Name: name, + Value: value, + }) + } + return pairs, nil +} + func toTags(labels types.Labels) []tags.Tag { t := []tags.Tag{} for n, v := range labels { @@ -391,25 +440,6 @@ func toHostEntryPtr(hosts types.HostsList) []ecs.TaskDefinition_HostEntry { return e } -func toKeyValuePair(environment types.MappingWithEquals) []ecs.TaskDefinition_KeyValuePair { - if environment == nil || len(environment) == 0 { - return nil - } - pairs := []ecs.TaskDefinition_KeyValuePair{} - for k, v := range environment { - name := k - var value string - if v != nil { - value = *v - } - pairs = append(pairs, ecs.TaskDefinition_KeyValuePair{ - Name: name, - Value: value, - }) - } - return pairs -} - func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name for key, value := range service.Extensions { diff --git a/ecs/pkg/amazon/backend/testdata/input/envfile b/ecs/pkg/amazon/backend/testdata/input/envfile new file mode 100644 index 00000000..6ac867af --- /dev/null +++ b/ecs/pkg/amazon/backend/testdata/input/envfile @@ -0,0 +1 @@ +FOO=BAR From d281f6cb3e97d384c208cfd043563f717f1fe810 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 11 Aug 2020 15:48:12 +0200 Subject: [PATCH 193/205] Allow fine tunning of awslogs Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 21 +++++++++---- ecs/pkg/amazon/backend/cloudformation_test.go | 31 +++++++++++++++++-- ecs/pkg/amazon/backend/compatibility.go | 12 +++++-- ecs/pkg/amazon/backend/convert.go | 29 ++++++++++++----- ecs/pkg/compose/x.go | 1 + 5 files changed, 76 insertions(+), 18 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index 482cf121..aa43dbe9 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -119,10 +119,7 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro project.Secrets[i] = s } - logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) - template.Resources["LogGroup"] = &logs.LogGroup{ - LogGroupName: logGroup, - } + createLogGroup(project, template) // Private DNS namespace will allow DNS name for the services to be ..local createCloudMap(project, template) @@ -190,7 +187,7 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro dependsOn = append(dependsOn, serviceResourceName(dependency)) } - minPercent, maxPercent, err := b.computeRollingUpdateLimits(service) + minPercent, maxPercent, err := computeRollingUpdateLimits(service) if err != nil { return nil, err } @@ -237,7 +234,19 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro return template, nil } -func (b Backend) computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) { +func createLogGroup(project *types.Project, template *cloudformation.Template) { + retention := 0 + if v, ok := project.Extensions[compose.ExtensionRetention]; ok { + retention = v.(int) + } + logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) + template.Resources["LogGroup"] = &logs.LogGroup{ + LogGroupName: logGroup, + RetentionInDays: retention, + } +} + +func computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) { maxPercent := 200 minPercent := 100 if service.Deploy == nil || service.Deploy.UpdateConfig == nil { diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index 84ddf530..f2a735c5 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -11,6 +11,7 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2" "github.com/awslabs/goformation/v4/cloudformation/iam" + "github.com/awslabs/goformation/v4/cloudformation/logs" "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" @@ -26,6 +27,25 @@ func TestSimpleConvert(t *testing.T) { golden.Assert(t, result, expected) } +func TestLogging(t *testing.T) { + template := convertYaml(t, "test", ` +services: + foo: + image: hello_world + logging: + options: + awslogs-datetime-pattern: "FOO" + +x-aws-logs_retention: 10 +`) + def := template.Resources["FooTaskDefinition"].(*ecs.TaskDefinition) + logging := def.ContainerDefinitions[0].LogConfiguration + assert.Equal(t, logging.Options["awslogs-datetime-pattern"], "FOO") + + logGroup := template.Resources["LogGroup"].(*logs.LogGroup) + assert.Equal(t, logGroup.RetentionInDays, 10) +} + func TestEnvFile(t *testing.T) { template := convertYaml(t, "test", ` services: @@ -36,8 +56,15 @@ services: `) def := template.Resources["FooTaskDefinition"].(*ecs.TaskDefinition) env := def.ContainerDefinitions[0].Environment - assert.Equal(t, env[0].Name, "FOO") - assert.Equal(t, env[0].Value, "BAR") + var found bool + for _, pair := range env { + if pair.Name == "FOO" { + assert.Equal(t, pair.Value, "BAR") + found = true + } + } + assert.Check(t, found, "environment variable FOO not set") + } func TestEnvFileAndEnv(t *testing.T) { diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/pkg/amazon/backend/compatibility.go index 02ef5e50..d217ca48 100644 --- a/ecs/pkg/amazon/backend/compatibility.go +++ b/ecs/pkg/amazon/backend/compatibility.go @@ -27,14 +27,16 @@ var compatibleComposeAttributes = []string{ "services.entrypoint", "services.environment", "services.env_file", - "service.image", - "services.init", "services.healthcheck", "services.healthcheck.interval", "services.healthcheck.retries", "services.healthcheck.start_period", "services.healthcheck.test", "services.healthcheck.timeout", + "services.image", + "services.init", + "services.logging", + "services.logging.options", "services.networks", "services.ports", "services.ports.mode", @@ -77,3 +79,9 @@ func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) } service.CapAdd = add } + +func (c *FargateCompatibilityChecker) CheckLoggingDriver(config *types.LoggingConfig) { + if config.Driver != "" && config.Driver != "awslogs" { + c.Unsupported("services.logging.driver %s is not supported", config.Driver) + } +} diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/pkg/amazon/backend/convert.go index b130d51a..2056dad4 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/pkg/amazon/backend/convert.go @@ -44,14 +44,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi fmt.Sprintf(" %s.local", project.Name), })) - logConfiguration := &ecs.TaskDefinition_LogConfiguration{ - LogDriver: ecsapi.LogDriverAwslogs, - Options: map[string]string{ - "awslogs-region": cloudformation.Ref("AWS::Region"), - "awslogs-group": cloudformation.Ref("LogGroup"), - "awslogs-stream-prefix": project.Name, - }, - } + logConfiguration := getLogConfiguration(service, project) var ( containers []ecs.TaskDefinition_ContainerDefinition @@ -222,6 +215,26 @@ func createEnvironment(project *types.Project, service types.ServiceConfig) ([]e return pairs, nil } +func getLogConfiguration(service types.ServiceConfig, project *types.Project) *ecs.TaskDefinition_LogConfiguration { + options := map[string]string{ + "awslogs-region": cloudformation.Ref("AWS::Region"), + "awslogs-group": cloudformation.Ref("LogGroup"), + "awslogs-stream-prefix": project.Name, + } + if service.Logging != nil { + for k, v := range service.Logging.Options { + if strings.HasPrefix(k, "awslogs-") { + options[k] = v + } + } + } + logConfiguration := &ecs.TaskDefinition_LogConfiguration{ + LogDriver: ecsapi.LogDriverAwslogs, + Options: options, + } + return logConfiguration +} + func toTags(labels types.Labels) []tags.Tag { t := []tags.Tag{} for n, v := range labels { diff --git a/ecs/pkg/compose/x.go b/ecs/pkg/compose/x.go index 44ebfae9..63f2e30e 100644 --- a/ecs/pkg/compose/x.go +++ b/ecs/pkg/compose/x.go @@ -9,4 +9,5 @@ const ( ExtensionKeys = "x-aws-keys" ExtensionMinPercent = "x-aws-min_percent" ExtensionMaxPercent = "x-aws-max_percent" + ExtensionRetention = "x-aws-logs_retention" ) From 83d65c02a0e73c9b452834a4202be55f97b4f7b4 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 12 Aug 2020 14:45:10 +0200 Subject: [PATCH 194/205] Fix flacky test Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/pkg/amazon/backend/cloudformation_test.go index f2a735c5..307b6036 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/pkg/amazon/backend/cloudformation_test.go @@ -64,7 +64,6 @@ services: } } assert.Check(t, found, "environment variable FOO not set") - } func TestEnvFileAndEnv(t *testing.T) { @@ -79,8 +78,14 @@ services: `) def := template.Resources["FooTaskDefinition"].(*ecs.TaskDefinition) env := def.ContainerDefinitions[0].Environment - assert.Equal(t, env[0].Name, "FOO") - assert.Equal(t, env[0].Value, "ZOT") + var found bool + for _, pair := range env { + if pair.Name == "FOO" { + assert.Equal(t, pair.Value, "ZOT") + found = true + } + } + assert.Check(t, found, "environment variable FOO not set") } func TestRollingUpdateLimits(t *testing.T) { From de99add26b71b9081b4fa85b87cda1ee6b2f4d3f Mon Sep 17 00:00:00 2001 From: aiordache Date: Tue, 11 Aug 2020 10:43:11 +0200 Subject: [PATCH 195/205] Use docker/api progress writer Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 14 ++- ecs/go.mod | 8 +- ecs/go.sum | 24 ++-- ecs/pkg/amazon/backend/backend.go | 9 ++ ecs/pkg/amazon/backend/down.go | 9 +- ecs/pkg/amazon/backend/logs.go | 10 +- ecs/pkg/amazon/backend/up.go | 35 +++++- ecs/pkg/amazon/backend/wait.go | 39 ++++++- ecs/pkg/compose/api.go | 5 +- ecs/pkg/console/colors.go | 9 ++ ecs/pkg/console/progress.go | 132 ---------------------- ecs/pkg/console/progress_test.go | 65 ----------- ecs/pkg/progress/plain.go | 29 +++++ ecs/pkg/progress/spinner.go | 50 +++++++++ ecs/pkg/progress/tty.go | 177 ++++++++++++++++++++++++++++++ ecs/pkg/progress/writer.go | 112 +++++++++++++++++++ 16 files changed, 491 insertions(+), 236 deletions(-) delete mode 100644 ecs/pkg/console/progress.go delete mode 100644 ecs/pkg/console/progress_test.go create mode 100644 ecs/pkg/progress/plain.go create mode 100644 ecs/pkg/progress/spinner.go create mode 100644 ecs/pkg/progress/tty.go create mode 100644 ecs/pkg/progress/writer.go diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index d0f1a224..c0ed6009 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -12,6 +12,7 @@ import ( amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" "github.com/docker/ecs-plugin/pkg/amazon/cloudformation" "github.com/docker/ecs-plugin/pkg/docker" + "github.com/docker/ecs-plugin/pkg/progress" "github.com/spf13/cobra" ) @@ -79,7 +80,11 @@ func UpCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { if err != nil { return err } - return backend.Up(context.Background(), opts) + + return progress.Run(context.Background(), func(ctx context.Context) error { + backend.SetWriter(ctx) + return backend.Up(ctx, opts) + }) }), } cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") @@ -124,7 +129,10 @@ func DownCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command if err != nil { return err } - return backend.Down(context.Background(), opts) + return progress.Run(context.Background(), func(ctx context.Context) error { + backend.SetWriter(ctx) + return backend.Down(ctx, opts) + }) }), } cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") @@ -139,7 +147,7 @@ func LogsCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command if err != nil { return err } - return backend.Logs(context.Background(), opts) + return backend.Logs(context.Background(), opts, os.Stdout) }), } return cmd diff --git a/ecs/go.mod b/ecs/go.mod index 594aa539..b6a1f1d2 100644 --- a/ecs/go.mod +++ b/ecs/go.mod @@ -1,7 +1,6 @@ module github.com/docker/ecs-plugin require ( - github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect github.com/Microsoft/hcsshim v0.8.7 // indirect github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect @@ -10,11 +9,13 @@ require ( github.com/bitly/go-hostpool v0.1.0 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 github.com/bugsnag/bugsnag-go v1.5.3 // indirect github.com/bugsnag/panicwrap v1.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cloudflare/cfssl v1.4.1 // indirect github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457 + github.com/containerd/console v1.0.0 github.com/containerd/containerd v1.3.2 // indirect github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 @@ -36,16 +37,17 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/miekg/pkcs11 v1.0.3 // indirect github.com/mitchellh/mapstructure v1.3.3 - github.com/morikuni/aec v1.0.0 // indirect + github.com/moby/term v0.0.0-20200611042045-63b9a826fb74 + github.com/morikuni/aec v1.0.0 github.com/onsi/ginkgo v1.11.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.6.0 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/theupdateframework/notary v0.6.1 // indirect github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/grpc v1.27.0 // indirect gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect diff --git a/ecs/go.sum b/ecs/go.sum index d73c6e3d..168b5633 100644 --- a/ecs/go.sum +++ b/ecs/go.sum @@ -34,6 +34,8 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 h1:gfAMKE626QEuKG3si0pdTRcr/YEbBoxY+3GOH3gWvl4= +github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129/go.mod h1:u9UyCz2eTrSGy6fbupqJ54eY5c4IC8gREQ1053dK12U= github.com/bugsnag/bugsnag-go v1.5.3 h1:yeRUT3mUE13jL1tGwvoQsKdVbAsQx9AJ+fqahKveP04= github.com/bugsnag/bugsnag-go v1.5.3/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA= @@ -54,12 +56,13 @@ github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiK github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo= github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= -github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3 h1:+ntlMTrEcScJjlnEOP8P1IIrusJaR93Eazr66YgUueA= -github.com/compose-spec/compose-go v0.0.0-20200716130117-e87e4f7839e3/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457 h1:8ely1LF7H02sIWz6QjgU53YBCiRpYlM9F9u1MeE1ZPk= github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457/go.mod h1:cS0vAvM6u9yjJgKWIH2yiqYMWO7WGJb+c0Irw+RefqU= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1 h1:uict5mhHFTzKLUCufdSLym7z/J0CbBJT59lYbP9wtbg= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v1.0.0 h1:fU3UuQapBs+zLJu82NhR11Rif1ny2zfMMAyPJzSN5tQ= +github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.2 h1:ForxmXkA6tPIvffbrDAcPUIB32QgXkt2XFj+F0UxetA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -76,6 +79,8 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -137,8 +142,7 @@ github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -157,8 +161,6 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -229,11 +231,11 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mjibson/esc v0.2.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= +github.com/moby/term v0.0.0-20200611042045-63b9a826fb74 h1:kvRIeqJNICemq2UFLx8q/Pj+1IRNZS0XPTaMFkuNsvg= +github.com/moby/term v0.0.0-20200611042045-63b9a826fb74/go.mod h1:pJ0Ot5YGdTcMdxnPMyGCfAr6fKXe0g9cDlz16MuFEBE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -401,7 +403,9 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -419,8 +423,8 @@ golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdO golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= diff --git a/ecs/pkg/amazon/backend/backend.go b/ecs/pkg/amazon/backend/backend.go index 4fd40c2b..caf7e17d 100644 --- a/ecs/pkg/amazon/backend/backend.go +++ b/ecs/pkg/amazon/backend/backend.go @@ -1,9 +1,12 @@ package backend import ( + "context" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/docker/ecs-plugin/pkg/amazon/sdk" + "github.com/docker/ecs-plugin/pkg/progress" ) func NewBackend(profile string, region string) (*Backend, error) { @@ -17,6 +20,7 @@ func NewBackend(profile string, region string) (*Backend, error) { if err != nil { return nil, err } + return &Backend{ Region: region, api: sdk.NewAPI(sess), @@ -26,4 +30,9 @@ func NewBackend(profile string, region string) (*Backend, error) { type Backend struct { Region string api sdk.API + writer progress.Writer +} + +func (b *Backend) SetWriter(context context.Context) { + b.writer = progress.ContextWriter(context) } diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/pkg/amazon/backend/down.go index fe89a9f2..0d235ad7 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/pkg/amazon/backend/down.go @@ -5,7 +5,6 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/pkg/console" ) func (b *Backend) Down(ctx context.Context, options *cli.ProjectOptions) error { @@ -18,13 +17,7 @@ func (b *Backend) Down(ctx context.Context, options *cli.ProjectOptions) error { if err != nil { return err } - - w := console.NewProgressWriter() - err = b.WaitStackCompletion(ctx, name, compose.StackDelete, w) - if err != nil { - return err - } - return nil + return b.WaitStackCompletion(ctx, name, compose.StackDelete) } func (b *Backend) projectName(options *cli.ProjectOptions) (string, error) { diff --git a/ecs/pkg/amazon/backend/logs.go b/ecs/pkg/amazon/backend/logs.go index ef5c61e2..fac65ba9 100644 --- a/ecs/pkg/amazon/backend/logs.go +++ b/ecs/pkg/amazon/backend/logs.go @@ -1,8 +1,10 @@ package backend import ( + "bytes" "context" "fmt" + "io" "os" "os/signal" "strconv" @@ -13,7 +15,7 @@ import ( "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) Logs(ctx context.Context, options *cli.ProjectOptions) error { +func (b *Backend) Logs(ctx context.Context, options *cli.ProjectOptions, writer io.Writer) error { name := options.Name if name == "" { project, err := cli.ProjectFromOptions(options) @@ -26,6 +28,7 @@ func (b *Backend) Logs(ctx context.Context, options *cli.ProjectOptions) error { err := b.api.GetLogs(ctx, name, &logConsumer{ colors: map[string]console.ColorFunc{}, width: 0, + writer: writer, }) if err != nil { return err @@ -45,8 +48,10 @@ func (l *logConsumer) Log(service, container, message string) { l.computeWidth() } prefix := fmt.Sprintf("%-"+strconv.Itoa(l.width)+"s |", service) + for _, line := range strings.Split(message, "\n") { - fmt.Printf("%s %s\n", cf(prefix), line) + buf := bytes.NewBufferString(fmt.Sprintf("%s %s\n", cf(prefix), line)) + l.writer.Write(buf.Bytes()) } } @@ -63,4 +68,5 @@ func (l *logConsumer) computeWidth() { type logConsumer struct { colors map[string]console.ColorFunc width int + writer io.Writer } diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 4bbe5bd0..38705e81 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -5,12 +5,13 @@ import ( "fmt" "os" "os/signal" + "strings" "syscall" "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/pkg/console" + "github.com/docker/ecs-plugin/pkg/progress" ) func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { @@ -82,10 +83,12 @@ func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { } } - fmt.Println() - w := console.NewProgressWriter() for k := range template.Resources { - w.ResourceEvent(k, "PENDING", "") + b.writer.Event(progress.Event{ + ID: k, + Status: progress.Working, + StatusText: "Pending", + }) } signalChan := make(chan os.Signal, 1) @@ -96,7 +99,29 @@ func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { b.Down(ctx, options) }() - return b.WaitStackCompletion(ctx, project.Name, operation, w) + err = b.WaitStackCompletion(ctx, project.Name, operation) + // update status for external resources (LB and cluster) + loadBalancerName := fmt.Sprintf("%.32s", fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name))) + for k := range template.Resources { + switch k { + case "Cluster": + if cluster == "" { + continue + } + case loadBalancerName: + if lb == "" { + continue + } + default: + continue + } + b.writer.Event(progress.Event{ + ID: k, + Status: progress.Done, + StatusText: "", + }) + } + return err } func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, error) { diff --git a/ecs/pkg/amazon/backend/wait.go b/ecs/pkg/amazon/backend/wait.go index babcaaf6..82b9863b 100644 --- a/ecs/pkg/amazon/backend/wait.go +++ b/ecs/pkg/amazon/backend/wait.go @@ -8,10 +8,11 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/docker/ecs-plugin/pkg/console" + "github.com/docker/ecs-plugin/pkg/compose" + "github.com/docker/ecs-plugin/pkg/progress" ) -func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int, w console.ProgressWriter) error { +func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int) error { knownEvents := map[string]struct{}{} // Get the unique Stack ID so we can collect events without getting some from previous deployments with same name @@ -22,7 +23,6 @@ func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operatio ticker := time.NewTicker(1 * time.Second) done := make(chan bool) - go func() { b.api.WaitStackComplete(ctx, stackID, operation) //nolint:errcheck ticker.Stop() @@ -55,11 +55,38 @@ func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operatio resource := aws.StringValue(event.LogicalResourceId) reason := aws.StringValue(event.ResourceStatusReason) status := aws.StringValue(event.ResourceStatus) - w.ResourceEvent(resource, status, reason) - if stackErr == nil && strings.HasSuffix(status, "_FAILED") { - stackErr = fmt.Errorf(reason) + progressStatus := progress.Working + + switch status { + case "CREATE_COMPLETE": + if operation == compose.StackCreate { + progressStatus = progress.Done + + } + case "UPDATE_COMPLETE": + if operation == compose.StackUpdate { + progressStatus = progress.Done + } + case "DELETE_COMPLETE": + if operation == compose.StackDelete { + progressStatus = progress.Done + } + default: + if strings.HasSuffix(status, "_FAILED") { + progressStatus = progress.Error + if stackErr == nil { + operation = compose.StackDelete + stackErr = fmt.Errorf(reason) + } + } } + b.writer.Event(progress.Event{ + ID: resource, + Status: progressStatus, + StatusText: status, + }) } } + return stackErr } diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go index 8e99b8ab..77f17055 100644 --- a/ecs/pkg/compose/api.go +++ b/ecs/pkg/compose/api.go @@ -2,6 +2,7 @@ package compose import ( "context" + "io" "github.com/awslabs/goformation/v4/cloudformation" "github.com/compose-spec/compose-go/cli" @@ -15,8 +16,8 @@ type API interface { CreateContextData(ctx context.Context, params map[string]string) (contextData interface{}, description string, err error) Convert(project *types.Project) (*cloudformation.Template, error) - Logs(ctx context.Context, options *cli.ProjectOptions) error - Ps(background context.Context, options *cli.ProjectOptions) ([]ServiceStatus, error) + Logs(ctx context.Context, options *cli.ProjectOptions, writer io.Writer) error + Ps(ctx context.Context, options *cli.ProjectOptions) ([]ServiceStatus, error) CreateSecret(ctx context.Context, secret Secret) (string, error) InspectSecret(ctx context.Context, id string) (Secret, error) diff --git a/ecs/pkg/console/colors.go b/ecs/pkg/console/colors.go index 517afb67..672b61f5 100644 --- a/ecs/pkg/console/colors.go +++ b/ecs/pkg/console/colors.go @@ -1,6 +1,7 @@ package console import ( + "fmt" "strconv" ) @@ -24,6 +25,14 @@ var Monochrome = func(s string) string { return s } +func ansiColor(code, s string) string { + return fmt.Sprintf("%s%s%s", ansi(code), s, ansi("0")) +} + +func ansi(code string) string { + return fmt.Sprintf("\033[%sm", code) +} + func makeColorFunc(code string) ColorFunc { return func(s string) string { return ansiColor(code, s) diff --git a/ecs/pkg/console/progress.go b/ecs/pkg/console/progress.go deleted file mode 100644 index 599ac5a7..00000000 --- a/ecs/pkg/console/progress.go +++ /dev/null @@ -1,132 +0,0 @@ -package console - -import ( - "fmt" - "io" - "os" - "strconv" - "strings" - - "github.com/sirupsen/logrus" -) - -type resource struct { - name string - status string - details string -} - -type progress struct { - console console - resources []*resource -} - -type ProgressWriter interface { - ResourceEvent(name string, status string, details string) -} - -func NewProgressWriter() ProgressWriter { - return &progress{ - console: ansiConsole{os.Stdout}, - } -} - -const ( - blue = "36;2" - red = "31;1" - green = "32;1" -) - -func (p *progress) ResourceEvent(name string, status string, details string) { - if logrus.IsLevelEnabled(logrus.DebugLevel) { - logrus.Debugf("> %s : %s %s\n", name, status, details) - return - } - p.console.MoveUp(len(p.resources)) - - newResource := true - for _, r := range p.resources { - if r.name == name { - newResource = false - r.status = status - r.details = details - break - } - } - if newResource { - p.resources = append(p.resources, &resource{name, status, details}) - } - - var width int - for _, r := range p.resources { - l := len(r.name) - if width < l { - width = l - } - } - - for _, r := range p.resources { - s := r.status - if strings.HasSuffix(s, "_IN_PROGRESS") { - s = p.console.WiP(s) - } else if strings.HasSuffix(s, "_COMPLETE") { - s = p.console.OK(s) - } else if strings.HasSuffix(s, "_FAILED") { - s = p.console.KO(s) - } - p.console.ClearLine() - p.console.Printf("%-"+strconv.Itoa(width)+"s ... %s %s", r.name, s, r.details) // nolint:errcheck - p.console.MoveDown(1) - } -} - -type console interface { - Printf(format string, a ...interface{}) - MoveUp(int) - MoveDown(int) - ClearLine() - OK(string) string - KO(string) string - WiP(string) string -} - -type ansiConsole struct { - out io.Writer -} - -func (c ansiConsole) Printf(format string, a ...interface{}) { - fmt.Fprintf(c.out, format, a...) // nolint:errcheck - fmt.Fprintf(c.out, "\r") -} - -func (c ansiConsole) MoveUp(i int) { - fmt.Fprintf(c.out, "\033[%dA", i) // nolint:errcheck -} - -func (c ansiConsole) MoveDown(i int) { - fmt.Fprintf(c.out, "\033[%dB", i) // nolint:errcheck -} - -func (c ansiConsole) ClearLine() { - fmt.Fprint(c.out, "\033[2K\r") // nolint:errcheck -} - -func (c ansiConsole) OK(s string) string { - return ansiColor(green, s) -} - -func (c ansiConsole) KO(s string) string { - return ansiColor(red, s) -} - -func (c ansiConsole) WiP(s string) string { - return ansiColor(blue, s) -} - -func ansiColor(code, s string) string { - return fmt.Sprintf("%s%s%s", ansi(code), s, ansi("0")) -} - -func ansi(code string) string { - return fmt.Sprintf("\033[%sm", code) -} diff --git a/ecs/pkg/console/progress_test.go b/ecs/pkg/console/progress_test.go deleted file mode 100644 index 552303c2..00000000 --- a/ecs/pkg/console/progress_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package console - -import ( - "fmt" - "testing" - - "gotest.tools/v3/assert" -) - -func TestProgressWriter(t *testing.T) { - c := &bufferConsole{} - p := progress{ - console: c, - } - p.ResourceEvent("resource1", "CREATE_IN_PROGRESS", "") - assert.Equal(t, c.lines[0], "resource1 ... CREATE_IN_PROGRESS ") - - p.ResourceEvent("resource2_long_name", "CREATE_IN_PROGRESS", "ok") - assert.Equal(t, c.lines[0], "resource1 ... CREATE_IN_PROGRESS ") - assert.Equal(t, c.lines[1], "resource2_long_name ... CREATE_IN_PROGRESS ok") - - p.ResourceEvent("resource2_long_name", "CREATE_COMPLETE", "done") - assert.Equal(t, c.lines[0], "resource1 ... CREATE_IN_PROGRESS ") - assert.Equal(t, c.lines[1], "resource2_long_name ... CREATE_COMPLETE done") - - p.ResourceEvent("resource1", "CREATE_FAILED", "oups") - assert.Equal(t, c.lines[0], "resource1 ... CREATE_FAILED oups") - assert.Equal(t, c.lines[1], "resource2_long_name ... CREATE_COMPLETE done") -} - -type bufferConsole struct { - pos int - lines []string -} - -func (b *bufferConsole) Printf(format string, a ...interface{}) { - b.lines[b.pos] = fmt.Sprintf(format, a...) -} - -func (b *bufferConsole) MoveUp(i int) { - b.pos -= i -} - -func (b *bufferConsole) MoveDown(i int) { - b.pos += i -} - -func (b *bufferConsole) ClearLine() { - if len(b.lines) <= b.pos { - b.lines = append(b.lines, "") - } - b.lines[b.pos] = "" -} - -func (b *bufferConsole) OK(s string) string { - return s -} - -func (b *bufferConsole) KO(s string) string { - return s -} - -func (b *bufferConsole) WiP(s string) string { - return s -} diff --git a/ecs/pkg/progress/plain.go b/ecs/pkg/progress/plain.go new file mode 100644 index 00000000..8e476807 --- /dev/null +++ b/ecs/pkg/progress/plain.go @@ -0,0 +1,29 @@ +package progress + +import ( + "context" + "fmt" + "io" +) + +type plainWriter struct { + out io.Writer + done chan bool +} + +func (p *plainWriter) Start(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-p.done: + return nil + } +} + +func (p *plainWriter) Event(e Event) { + fmt.Println(e.ID, e.Text, e.StatusText) +} + +func (p *plainWriter) Stop() { + p.done <- true +} diff --git a/ecs/pkg/progress/spinner.go b/ecs/pkg/progress/spinner.go new file mode 100644 index 00000000..695a5642 --- /dev/null +++ b/ecs/pkg/progress/spinner.go @@ -0,0 +1,50 @@ +package progress + +import ( + "runtime" + "time" +) + +type spinner struct { + time time.Time + index int + chars []string + stop bool + done string +} + +func newSpinner() *spinner { + chars := []string{ + "⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏", + } + done := "⠿" + + if runtime.GOOS == "windows" { + chars = []string{"-"} + done = "-" + } + + return &spinner{ + index: 0, + time: time.Now(), + chars: chars, + done: done, + } +} + +func (s *spinner) String() string { + if s.stop { + return s.done + } + + d := time.Since(s.time) + if d.Milliseconds() > 100 { + s.index = (s.index + 1) % len(s.chars) + } + + return s.chars[s.index] +} + +func (s *spinner) Stop() { + s.stop = true +} diff --git a/ecs/pkg/progress/tty.go b/ecs/pkg/progress/tty.go new file mode 100644 index 00000000..6877e096 --- /dev/null +++ b/ecs/pkg/progress/tty.go @@ -0,0 +1,177 @@ +package progress + +import ( + "context" + "fmt" + "io" + "runtime" + "strings" + "sync" + "time" + + "github.com/buger/goterm" + "github.com/morikuni/aec" +) + +type ttyWriter struct { + out io.Writer + events map[string]Event + eventIDs []string + repeated bool + numLines int + done chan bool + mtx *sync.RWMutex +} + +func (w *ttyWriter) Start(ctx context.Context) error { + ticker := time.NewTicker(100 * time.Millisecond) + + for { + select { + case <-ctx.Done(): + w.print() + return ctx.Err() + case <-w.done: + w.print() + return nil + case <-ticker.C: + w.print() + } + } +} + +func (w *ttyWriter) Stop() { + w.done <- true +} + +func (w *ttyWriter) Event(e Event) { + w.mtx.Lock() + defer w.mtx.Unlock() + if !StringContains(w.eventIDs, e.ID) { + w.eventIDs = append(w.eventIDs, e.ID) + } + if _, ok := w.events[e.ID]; ok { + last := w.events[e.ID] + switch e.Status { + case Done, Error: + if last.Status != e.Status { + last.stop() + } + } + last.Status = e.Status + last.Text = e.Text + last.StatusText = e.StatusText + w.events[e.ID] = last + } else { + e.startTime = time.Now() + e.spinner = newSpinner() + w.events[e.ID] = e + } +} + +func (w *ttyWriter) print() { + w.mtx.Lock() + defer w.mtx.Unlock() + if len(w.eventIDs) == 0 { + return + } + terminalWidth := goterm.Width() + b := aec.EmptyBuilder + for i := 0; i <= w.numLines; i++ { + b = b.Up(1) + } + if !w.repeated { + b = b.Down(1) + } + w.repeated = true + fmt.Fprint(w.out, b.Column(0).ANSI) + + // Hide the cursor while we are printing + fmt.Fprint(w.out, aec.Hide) + defer fmt.Fprint(w.out, aec.Show) + + firstLine := fmt.Sprintf("[+] Running %d/%d", numDone(w.events), w.numLines) + if w.numLines != 0 && numDone(w.events) == w.numLines { + firstLine = aec.Apply(firstLine, aec.BlueF) + } + fmt.Fprintln(w.out, firstLine) + + var statusPadding int + for _, v := range w.eventIDs { + l := len(fmt.Sprintf("%s %s", w.events[v].ID, w.events[v].Text)) + if statusPadding < l { + statusPadding = l + } + } + + numLines := 0 + for _, v := range w.eventIDs { + line := lineText(w.events[v], terminalWidth, statusPadding, runtime.GOOS != "windows") + // nolint: errcheck + fmt.Fprint(w.out, line) + numLines++ + } + + w.numLines = numLines +} + +func lineText(event Event, terminalWidth, statusPadding int, color bool) string { + endTime := time.Now() + if event.Status != Working { + endTime = event.endTime + } + + elapsed := endTime.Sub(event.startTime).Seconds() + + textLen := len(fmt.Sprintf("%s %s", event.ID, event.Text)) + padding := statusPadding - textLen + if padding < 0 { + padding = 0 + } + text := fmt.Sprintf(" %s %s %s%s %s", + event.spinner.String(), + event.ID, + event.Text, + strings.Repeat(" ", padding), + event.StatusText, + ) + timer := fmt.Sprintf("%.1fs\n", elapsed) + o := align(text, timer, terminalWidth) + + if color { + color := aec.WhiteF + if event.Status == Done { + color = aec.BlueF + } + if event.Status == Error { + color = aec.RedF + } + return aec.Apply(o, color) + } + + return o +} + +func numDone(events map[string]Event) int { + i := 0 + for _, e := range events { + if e.Status == Done { + i++ + } + } + return i +} + +func align(l, r string, w int) string { + return fmt.Sprintf("%-[2]*[1]s %[3]s", l, w-len(r)-1, r) +} + +// StringContains check if an array contains a specific value +func StringContains(array []string, needle string) bool { + for _, val := range array { + if val == needle { + return true + } + } + return false +} diff --git a/ecs/pkg/progress/writer.go b/ecs/pkg/progress/writer.go new file mode 100644 index 00000000..a5b8e1cd --- /dev/null +++ b/ecs/pkg/progress/writer.go @@ -0,0 +1,112 @@ +package progress + +import ( + "context" + "os" + "sync" + "time" + + "github.com/containerd/console" + "github.com/moby/term" + "golang.org/x/sync/errgroup" +) + +// EventStatus indicates the status of an action +type EventStatus int + +const ( + // Working means that the current task is working + Working EventStatus = iota + // Done means that the current task is done + Done + // Error means that the current task has errored + Error +) + +// Event reprensents a progress event +type Event struct { + ID string + Text string + Status EventStatus + StatusText string + Done bool + + startTime time.Time + endTime time.Time + spinner *spinner +} + +func (e *Event) stop() { + e.endTime = time.Now() + e.spinner.Stop() +} + +// Writer can write multiple progress events +type Writer interface { + Start(context.Context) error + Stop() + Event(Event) +} + +type writerKey struct{} + +// WithContextWriter adds the writer to the context +func WithContextWriter(ctx context.Context, writer Writer) context.Context { + return context.WithValue(ctx, writerKey{}, writer) +} + +// ContextWriter returns the writer from the context +func ContextWriter(ctx context.Context) Writer { + s, _ := ctx.Value(writerKey{}).(Writer) + return s +} + +type progressFunc func(context.Context) error + +// Run will run a writer and the progress function +// in parallel +func Run(ctx context.Context, pf progressFunc) error { + eg, _ := errgroup.WithContext(ctx) + w, err := NewWriter(os.Stderr) + if err != nil { + return err + } + eg.Go(func() error { + return w.Start(context.Background()) + }) + + ctx = WithContextWriter(ctx, w) + + eg.Go(func() error { + defer w.Stop() + return pf(ctx) + }) + + return eg.Wait() +} + +// NewWriter returns a new multi-progress writer +func NewWriter(out console.File) (Writer, error) { + _, isTerminal := term.GetFdInfo(out) + + if isTerminal { + con, err := console.ConsoleFromFile(out) + if err != nil { + return nil, err + } + + return &ttyWriter{ + out: con, + eventIDs: []string{}, + events: map[string]Event{}, + repeated: false, + done: make(chan bool), + mtx: &sync.RWMutex{}, + }, nil + } + + return &plainWriter{ + out: out, + done: make(chan bool), + }, nil +} From 8182c98abf0b4cca6f1766a05a06ca668ada0a03 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 13 Aug 2020 10:27:01 +0200 Subject: [PATCH 196/205] Don't pretend we know resources to be created some resources are controlled by a CloudFormation Condition and as such won't be created. If we add them to the progresswriter, the latter will never receive status update. Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/up.go | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/pkg/amazon/backend/up.go index 38705e81..098caaac 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/pkg/amazon/backend/up.go @@ -5,13 +5,11 @@ import ( "fmt" "os" "os/signal" - "strings" "syscall" "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/pkg/progress" ) func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { @@ -83,14 +81,6 @@ func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { } } - for k := range template.Resources { - b.writer.Event(progress.Event{ - ID: k, - Status: progress.Working, - StatusText: "Pending", - }) - } - signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) go func() { @@ -100,27 +90,6 @@ func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { }() err = b.WaitStackCompletion(ctx, project.Name, operation) - // update status for external resources (LB and cluster) - loadBalancerName := fmt.Sprintf("%.32s", fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name))) - for k := range template.Resources { - switch k { - case "Cluster": - if cluster == "" { - continue - } - case loadBalancerName: - if lb == "" { - continue - } - default: - continue - } - b.writer.Event(progress.Event{ - ID: k, - Status: progress.Done, - StatusText: "", - }) - } return err } From f74cc8f0aab03a6fe979cae4794a5e7a40d9933a Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 13 Aug 2020 15:43:24 +0200 Subject: [PATCH 197/205] Allow user to customize Roles / ManagedPolicy Signed-off-by: Nicolas De Loof --- ecs/pkg/amazon/backend/cloudformation.go | 19 +++++++++++++++---- ecs/pkg/amazon/backend/iam.go | 2 +- ecs/pkg/compose/x.go | 2 ++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/pkg/amazon/backend/cloudformation.go index aa43dbe9..c6c033db 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/pkg/amazon/backend/cloudformation.go @@ -440,15 +440,26 @@ func createTaskExecutionRole(service types.ServiceConfig, err error, definition PolicyDocument: policy, PolicyName: fmt.Sprintf("%sGrantAccessToSecrets", service.Name), }) + } + if roles, ok := service.Extensions[compose.ExtensionRole]; ok { + rolePolicies = append(rolePolicies, iam.Role_Policy{ + PolicyDocument: roles, + }) + } + managedPolicies := []string{ + ECSTaskExecutionPolicy, + ECRReadOnlyPolicy, + } + if v, ok := service.Extensions[compose.ExtensionManagedPolicies]; ok { + for _, s := range v.([]interface{}) { + managedPolicies = append(managedPolicies, s.(string)) + } } template.Resources[taskExecutionRole] = &iam.Role{ AssumeRolePolicyDocument: assumeRolePolicyDocument, Policies: rolePolicies, - ManagedPolicyArns: []string{ - ECSTaskExecutionPolicy, - ECRReadOnlyPolicy, - }, + ManagedPolicyArns: managedPolicies, } return taskExecutionRole, nil } diff --git a/ecs/pkg/amazon/backend/iam.go b/ecs/pkg/amazon/backend/iam.go index 81a4fdb0..4b282020 100644 --- a/ecs/pkg/amazon/backend/iam.go +++ b/ecs/pkg/amazon/backend/iam.go @@ -22,7 +22,7 @@ var assumeRolePolicyDocument = PolicyDocument{ }, } -// could alternatively depend on https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/master/pkg/cloud/services/iam/types.go#L52 +// could alternatively depend on https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/master/cmd/clusterawsadm/api/iam/v1alpha1/types.go type PolicyDocument struct { Version string `json:",omitempty"` Statement []PolicyStatement `json:",omitempty"` diff --git a/ecs/pkg/compose/x.go b/ecs/pkg/compose/x.go index 63f2e30e..3a4ffdb6 100644 --- a/ecs/pkg/compose/x.go +++ b/ecs/pkg/compose/x.go @@ -10,4 +10,6 @@ const ( ExtensionMinPercent = "x-aws-min_percent" ExtensionMaxPercent = "x-aws-max_percent" ExtensionRetention = "x-aws-logs_retention" + ExtensionRole = "x-aws-role" + ExtensionManagedPolicies = "x-aws-policies" ) From 9eb0a10517744d762f11d6bc06962ae1cc0a08cc Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 14 Aug 2020 12:01:22 +0200 Subject: [PATCH 198/205] Init progress writer in WaitStackCompletion Signed-off-by: aiordache Signed-off-by: Nicolas De Loof --- ecs/cmd/commands/compose.go | 2 -- ecs/pkg/amazon/backend/backend.go | 8 -------- ecs/pkg/amazon/backend/wait.go | 5 +++-- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go index c0ed6009..7fea85e9 100644 --- a/ecs/cmd/commands/compose.go +++ b/ecs/cmd/commands/compose.go @@ -82,7 +82,6 @@ func UpCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { } return progress.Run(context.Background(), func(ctx context.Context) error { - backend.SetWriter(ctx) return backend.Up(ctx, opts) }) }), @@ -130,7 +129,6 @@ func DownCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command return err } return progress.Run(context.Background(), func(ctx context.Context) error { - backend.SetWriter(ctx) return backend.Down(ctx, opts) }) }), diff --git a/ecs/pkg/amazon/backend/backend.go b/ecs/pkg/amazon/backend/backend.go index caf7e17d..2b265f3e 100644 --- a/ecs/pkg/amazon/backend/backend.go +++ b/ecs/pkg/amazon/backend/backend.go @@ -1,12 +1,9 @@ package backend import ( - "context" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/docker/ecs-plugin/pkg/amazon/sdk" - "github.com/docker/ecs-plugin/pkg/progress" ) func NewBackend(profile string, region string) (*Backend, error) { @@ -30,9 +27,4 @@ func NewBackend(profile string, region string) (*Backend, error) { type Backend struct { Region string api sdk.API - writer progress.Writer -} - -func (b *Backend) SetWriter(context context.Context) { - b.writer = progress.ContextWriter(context) } diff --git a/ecs/pkg/amazon/backend/wait.go b/ecs/pkg/amazon/backend/wait.go index 82b9863b..dcc26772 100644 --- a/ecs/pkg/amazon/backend/wait.go +++ b/ecs/pkg/amazon/backend/wait.go @@ -14,7 +14,8 @@ import ( func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int) error { knownEvents := map[string]struct{}{} - + // progress writer + w := progress.ContextWriter(ctx) // Get the unique Stack ID so we can collect events without getting some from previous deployments with same name stackID, err := b.api.GetStackID(ctx, name) if err != nil { @@ -80,7 +81,7 @@ func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operatio } } } - b.writer.Event(progress.Event{ + w.Event(progress.Event{ ID: resource, Status: progressStatus, StatusText: status, From e02d8d549b258e454a0ba94267692434c6660e32 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 17 Aug 2020 17:48:52 +0200 Subject: [PATCH 199/205] Make ECS integration a compose-cli backend Signed-off-by: Nicolas De Loof --- aci/backend.go | 15 +- cli/cmd/compose/down.go | 4 +- cli/cmd/compose/logs.go | 7 +- cli/cmd/compose/ps.go | 4 +- cli/cmd/compose/up.go | 4 +- compose/api.go | 27 +- {ecs/pkg/compose => compose}/tags.go | 0 ecs/Dockerfile | 57 -- ecs/LICENSE | 191 ------- ecs/Makefile | 48 -- ecs/README.md | 110 ---- ecs/backend.go | 27 +- ecs/builder.Makefile | 39 -- .../amazon/backend => }/cloudformation.go | 18 +- .../backend => }/cloudformation_test.go | 3 +- ecs/cmd/commands/compose.go | 152 ------ ecs/cmd/commands/context.go | 31 -- ecs/cmd/commands/opts.go | 28 - ecs/cmd/commands/root.go | 32 -- ecs/cmd/commands/root_test.go | 13 - ecs/cmd/commands/secret.go | 140 ----- ecs/cmd/commands/setup.go | 262 --------- ecs/cmd/commands/version.go | 20 - ecs/cmd/commands/version_test.go | 20 - ecs/cmd/main/main.go | 23 - ecs/{pkg/console => }/colors.go | 2 +- ecs/{pkg/amazon/backend => }/compatibility.go | 2 +- ecs/context.go | 2 - ecs/{pkg/amazon/backend => }/convert.go | 9 +- ecs/doc.go | 14 - ecs/docs/get-started-linux.md | 82 --- ecs/docs/requirements.md | 31 -- ecs/{pkg/amazon/backend => }/down.go | 11 +- ecs/example/Makefile | 25 - ecs/example/README.md | 181 ------- ecs/example/app/Dockerfile | 7 - ecs/example/app/app.py | 19 - ecs/example/app/requirements.txt | 2 - ecs/example/app/scripts/entrypoint.sh | 4 - ecs/example/app/templates/index.html | 125 ----- ecs/example/docker-compose.yml | 12 - ecs/go.mod | 61 --- ecs/go.sum | 496 ------------------ ecs/golangci.yaml | 12 - ecs/{pkg/amazon/backend => }/iam.go | 2 +- ecs/internal/version.go | 8 - ecs/{pkg/amazon/backend => }/list.go | 12 +- ecs/{pkg/amazon/backend => }/logs.go | 14 +- .../amazon/cloudformation => }/marshall.go | 2 +- ecs/pkg/amazon/amazon.go | 8 - ecs/pkg/amazon/backend/backend.go | 30 -- ecs/pkg/amazon/backend/context.go | 33 -- ecs/pkg/amazon/backend/secrets.go | 23 - ecs/pkg/amazon/sdk/api.go | 42 -- ecs/pkg/compose/api.go | 26 - .../simple/compose-with-overrides.yaml | 4 - ecs/pkg/compose/testdata/simple/compose.yaml | 4 - ecs/pkg/docker/contextStore.go | 76 --- ecs/pkg/progress/plain.go | 29 - ecs/pkg/progress/spinner.go | 50 -- ecs/pkg/progress/tty.go | 177 ------- ecs/pkg/progress/writer.go | 112 ---- ecs/{pkg/amazon/sdk => }/sdk.go | 45 +- ecs/secrets.go | 21 + ecs/secrets/Dockerfile | 2 +- ecs/secrets/main/main.go | 3 +- .../backend => }/testdata/input/envfile | 0 .../testdata/input/simple-single-service.yaml | 0 .../testdata/invalid_network_mode.yaml | 0 .../simple-cloudformation-conversion.golden | 0 ecs/tests/e2e_deploy_services_test.go | 65 --- ecs/tests/main_test.go | 126 ----- .../testdata/input/simple-single-service.yaml | 6 - ecs/tests/testdata/plugin-usage.golden | 14 - ecs/tests/version_test.go | 18 - ecs/{pkg/compose => }/types.go | 18 +- ecs/{pkg/amazon/backend => }/up.go | 41 +- ecs/{pkg/amazon/backend => }/wait.go | 21 +- ecs/{pkg/compose => }/x.go | 2 +- example/backend.go | 14 +- go.mod | 8 +- go.sum | 175 +----- 82 files changed, 180 insertions(+), 3423 deletions(-) rename {ecs/pkg/compose => compose}/tags.go (100%) delete mode 100644 ecs/Dockerfile delete mode 100644 ecs/LICENSE delete mode 100644 ecs/Makefile delete mode 100644 ecs/README.md delete mode 100644 ecs/builder.Makefile rename ecs/{pkg/amazon/backend => }/cloudformation.go (97%) rename ecs/{pkg/amazon/backend => }/cloudformation_test.go (99%) delete mode 100644 ecs/cmd/commands/compose.go delete mode 100644 ecs/cmd/commands/context.go delete mode 100644 ecs/cmd/commands/opts.go delete mode 100644 ecs/cmd/commands/root.go delete mode 100644 ecs/cmd/commands/root_test.go delete mode 100644 ecs/cmd/commands/secret.go delete mode 100644 ecs/cmd/commands/setup.go delete mode 100644 ecs/cmd/commands/version.go delete mode 100644 ecs/cmd/commands/version_test.go delete mode 100644 ecs/cmd/main/main.go rename ecs/{pkg/console => }/colors.go (98%) rename ecs/{pkg/amazon/backend => }/compatibility.go (99%) rename ecs/{pkg/amazon/backend => }/convert.go (98%) delete mode 100644 ecs/doc.go delete mode 100644 ecs/docs/get-started-linux.md delete mode 100644 ecs/docs/requirements.md rename ecs/{pkg/amazon/backend => }/down.go (52%) delete mode 100644 ecs/example/Makefile delete mode 100644 ecs/example/README.md delete mode 100644 ecs/example/app/Dockerfile delete mode 100644 ecs/example/app/app.py delete mode 100644 ecs/example/app/requirements.txt delete mode 100755 ecs/example/app/scripts/entrypoint.sh delete mode 100644 ecs/example/app/templates/index.html delete mode 100644 ecs/example/docker-compose.yml delete mode 100644 ecs/go.mod delete mode 100644 ecs/go.sum delete mode 100644 ecs/golangci.yaml rename ecs/{pkg/amazon/backend => }/iam.go (98%) delete mode 100644 ecs/internal/version.go rename ecs/{pkg/amazon/backend => }/list.go (71%) rename ecs/{pkg/amazon/backend => }/logs.go (77%) rename ecs/{pkg/amazon/cloudformation => }/marshall.go (98%) delete mode 100644 ecs/pkg/amazon/amazon.go delete mode 100644 ecs/pkg/amazon/backend/backend.go delete mode 100644 ecs/pkg/amazon/backend/context.go delete mode 100644 ecs/pkg/amazon/backend/secrets.go delete mode 100644 ecs/pkg/amazon/sdk/api.go delete mode 100644 ecs/pkg/compose/api.go delete mode 100644 ecs/pkg/compose/testdata/simple/compose-with-overrides.yaml delete mode 100644 ecs/pkg/compose/testdata/simple/compose.yaml delete mode 100644 ecs/pkg/docker/contextStore.go delete mode 100644 ecs/pkg/progress/plain.go delete mode 100644 ecs/pkg/progress/spinner.go delete mode 100644 ecs/pkg/progress/tty.go delete mode 100644 ecs/pkg/progress/writer.go rename ecs/{pkg/amazon/sdk => }/sdk.go (93%) create mode 100644 ecs/secrets.go rename ecs/{pkg/amazon/backend => }/testdata/input/envfile (100%) rename ecs/{pkg/amazon/backend => }/testdata/input/simple-single-service.yaml (100%) rename ecs/{pkg/amazon/backend => }/testdata/invalid_network_mode.yaml (100%) rename ecs/{pkg/amazon/backend => }/testdata/simple/simple-cloudformation-conversion.golden (100%) delete mode 100644 ecs/tests/e2e_deploy_services_test.go delete mode 100644 ecs/tests/main_test.go delete mode 100644 ecs/tests/testdata/input/simple-single-service.yaml delete mode 100644 ecs/tests/testdata/plugin-usage.golden delete mode 100644 ecs/tests/version_test.go rename ecs/{pkg/compose => }/types.go (78%) rename ecs/{pkg/amazon/backend => }/up.go (65%) rename ecs/{pkg/amazon/backend => }/wait.go (75%) rename ecs/{pkg/compose => }/x.go (97%) diff --git a/aci/backend.go b/aci/backend.go index 71679a07..fc5fe758 100644 --- a/aci/backend.go +++ b/aci/backend.go @@ -19,6 +19,7 @@ package aci import ( "context" "fmt" + "io" "net/http" "strconv" "strings" @@ -31,8 +32,6 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - ecstypes "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/api/aci/convert" "github.com/docker/api/aci/login" "github.com/docker/api/backend" @@ -404,8 +403,8 @@ type aciComposeService struct { ctx store.AciContext } -func (cs *aciComposeService) Up(ctx context.Context, opts cli.ProjectOptions) error { - project, err := cli.ProjectFromOptions(&opts) +func (cs *aciComposeService) Up(ctx context.Context, opts *cli.ProjectOptions) error { + project, err := cli.ProjectFromOptions(opts) if err != nil { return err } @@ -419,13 +418,13 @@ func (cs *aciComposeService) Up(ctx context.Context, opts cli.ProjectOptions) er return createOrUpdateACIContainers(ctx, cs.ctx, groupDefinition) } -func (cs *aciComposeService) Down(ctx context.Context, opts cli.ProjectOptions) error { +func (cs *aciComposeService) Down(ctx context.Context, opts *cli.ProjectOptions) error { var project types.Project if opts.Name != "" { project = types.Project{Name: opts.Name} } else { - fullProject, err := cli.ProjectFromOptions(&opts) + fullProject, err := cli.ProjectFromOptions(opts) if err != nil { return err } @@ -444,11 +443,11 @@ func (cs *aciComposeService) Down(ctx context.Context, opts cli.ProjectOptions) return err } -func (cs *aciComposeService) Ps(ctx context.Context, opts cli.ProjectOptions) ([]ecstypes.ServiceStatus, error) { +func (cs *aciComposeService) Ps(ctx context.Context, opts *cli.ProjectOptions) ([]compose.ServiceStatus, error) { return nil, errdefs.ErrNotImplemented } -func (cs *aciComposeService) Logs(ctx context.Context, opts cli.ProjectOptions) error { +func (cs *aciComposeService) Logs(ctx context.Context, opts *cli.ProjectOptions, w io.Writer) error { return errdefs.ErrNotImplemented } diff --git a/cli/cmd/compose/down.go b/cli/cmd/compose/down.go index 20544358..7389dadc 100644 --- a/cli/cmd/compose/down.go +++ b/cli/cmd/compose/down.go @@ -31,7 +31,7 @@ func downCommand() *cobra.Command { downCmd := &cobra.Command{ Use: "down", RunE: func(cmd *cobra.Command, args []string) error { - return runDown(cmd.Context(), opts) + return runDown(cmd.Context(), &opts) }, } downCmd.Flags().StringVarP(&opts.Name, "project-name", "p", "", "Project name") @@ -41,7 +41,7 @@ func downCommand() *cobra.Command { return downCmd } -func runDown(ctx context.Context, opts cli.ProjectOptions) error { +func runDown(ctx context.Context, opts *cli.ProjectOptions) error { c, err := client.New(ctx) if err != nil { return err diff --git a/cli/cmd/compose/logs.go b/cli/cmd/compose/logs.go index 15e0a260..84ddda0e 100644 --- a/cli/cmd/compose/logs.go +++ b/cli/cmd/compose/logs.go @@ -19,6 +19,7 @@ package compose import ( "context" "errors" + "os" "github.com/compose-spec/compose-go/cli" "github.com/spf13/cobra" @@ -31,7 +32,7 @@ func logsCommand() *cobra.Command { logsCmd := &cobra.Command{ Use: "logs", RunE: func(cmd *cobra.Command, args []string) error { - return runLogs(cmd.Context(), opts) + return runLogs(cmd.Context(), &opts) }, } logsCmd.Flags().StringVarP(&opts.Name, "project-name", "p", "", "Project name") @@ -41,7 +42,7 @@ func logsCommand() *cobra.Command { return logsCmd } -func runLogs(ctx context.Context, opts cli.ProjectOptions) error { +func runLogs(ctx context.Context, opts *cli.ProjectOptions) error { c, err := client.New(ctx) if err != nil { return err @@ -52,5 +53,5 @@ func runLogs(ctx context.Context, opts cli.ProjectOptions) error { return errors.New("compose not implemented in current context") } - return composeService.Logs(ctx, opts) + return composeService.Logs(ctx, opts, os.Stdout) } diff --git a/cli/cmd/compose/ps.go b/cli/cmd/compose/ps.go index e5930740..ee152e23 100644 --- a/cli/cmd/compose/ps.go +++ b/cli/cmd/compose/ps.go @@ -36,7 +36,7 @@ func psCommand() *cobra.Command { psCmd := &cobra.Command{ Use: "ps", RunE: func(cmd *cobra.Command, args []string) error { - return runPs(cmd.Context(), opts) + return runPs(cmd.Context(), &opts) }, } psCmd.Flags().StringVarP(&opts.Name, "project-name", "p", "", "Project name") @@ -46,7 +46,7 @@ func psCommand() *cobra.Command { return psCmd } -func runPs(ctx context.Context, opts cli.ProjectOptions) error { +func runPs(ctx context.Context, opts *cli.ProjectOptions) error { c, err := client.New(ctx) if err != nil { return err diff --git a/cli/cmd/compose/up.go b/cli/cmd/compose/up.go index 4ac19a43..91e7bf25 100644 --- a/cli/cmd/compose/up.go +++ b/cli/cmd/compose/up.go @@ -32,7 +32,7 @@ func upCommand() *cobra.Command { upCmd := &cobra.Command{ Use: "up", RunE: func(cmd *cobra.Command, args []string) error { - return runUp(cmd.Context(), opts) + return runUp(cmd.Context(), &opts) }, } upCmd.Flags().StringVarP(&opts.Name, "project-name", "p", "", "Project name") @@ -44,7 +44,7 @@ func upCommand() *cobra.Command { return upCmd } -func runUp(ctx context.Context, opts cli.ProjectOptions) error { +func runUp(ctx context.Context, opts *cli.ProjectOptions) error { c, err := client.New(ctx) if err != nil { return err diff --git a/compose/api.go b/compose/api.go index a9592d69..d80050f0 100644 --- a/compose/api.go +++ b/compose/api.go @@ -18,19 +18,36 @@ package compose import ( "context" + "io" "github.com/compose-spec/compose-go/cli" - types "github.com/docker/ecs-plugin/pkg/compose" ) // Service manages a compose project type Service interface { // Up executes the equivalent to a `compose up` - Up(ctx context.Context, opts cli.ProjectOptions) error + Up(ctx context.Context, opts *cli.ProjectOptions) error // Down executes the equivalent to a `compose down` - Down(ctx context.Context, opts cli.ProjectOptions) error + Down(ctx context.Context, opts *cli.ProjectOptions) error // Logs executes the equivalent to a `compose logs` - Logs(ctx context.Context, opts cli.ProjectOptions) error + Logs(ctx context.Context, opts *cli.ProjectOptions, w io.Writer) error // Ps executes the equivalent to a `compose ps` - Ps(ctx context.Context, opts cli.ProjectOptions) ([]types.ServiceStatus, error) + Ps(ctx context.Context, opts *cli.ProjectOptions) ([]ServiceStatus, error) } + + +type LoadBalancer struct { + URL string + TargetPort int + PublishedPort int + Protocol string +} + +type ServiceStatus struct { + ID string + Name string + Replicas int + Desired int + Ports []string + LoadBalancers []LoadBalancer +} \ No newline at end of file diff --git a/ecs/pkg/compose/tags.go b/compose/tags.go similarity index 100% rename from ecs/pkg/compose/tags.go rename to compose/tags.go diff --git a/ecs/Dockerfile b/ecs/Dockerfile deleted file mode 100644 index e5b506b9..00000000 --- a/ecs/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -# syntax = docker/dockerfile:experimental -ARG GO_VERSION=1.14.4-alpine -ARG ALPINE_PKG_DOCKER_VERSION=19.03.12-r0 -ARG GOLANGCI_LINT_VERSION=v1.27.0-alpine - -FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS base -WORKDIR /ecs-plugin -ENV GO111MODULE=on -ARG ALPINE_PKG_DOCKER_VERSION -RUN apk add --no-cache \ - docker=${ALPINE_PKG_DOCKER_VERSION} \ - make \ - build-base -COPY go.* . -RUN --mount=type=cache,target=/go/pkg/mod \ - go mod download - -FROM base AS make-plugin -ARG TARGETOS -ARG TARGETARCH -ARG COMMIT -ARG TAG -COPY . . -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - GOOS=${TARGETOS} \ - GOARCH=${TARGETARCH} \ - make -f builder.Makefile build - -FROM base AS make-cross -ARG COMMIT -ARG TAG -COPY . . -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - make -f builder.Makefile cross - -FROM scratch AS build -COPY --from=make-plugin /ecs-plugin/dist/docker-ecs . - -FROM scratch AS cross -COPY --from=make-cross /ecs-plugin/dist/* . - -FROM make-plugin AS test -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - make -f builder.Makefile test - -FROM golangci/golangci-lint:${GOLANGCI_LINT_VERSION} AS lint-base - -FROM base AS lint -COPY --from=lint-base /usr/bin/golangci-lint /usr/bin/golangci-lint -RUN --mount=target=. \ - --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - --mount=type=cache,target=/root/.cache/golangci-lint \ - make -f builder.Makefile lint diff --git a/ecs/LICENSE b/ecs/LICENSE deleted file mode 100644 index 6d8d58fb..00000000 --- a/ecs/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - - Apache License - Version 2.0, January 2004 - https://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2013-2018 Docker, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/ecs/Makefile b/ecs/Makefile deleted file mode 100644 index 10932586..00000000 --- a/ecs/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -PLATFORM?=local -PWD=$(shell pwd) - -export DOCKER_BUILDKIT=1 - -COMMIT := $(shell git rev-parse --short HEAD) -TAG := $(shell git describe --tags --dirty --match "v*") - -.DEFAULT_GOAL := build - -build: ## Build for the current - @docker build . \ - --output ./dist \ - --platform ${PLATFORM} \ - --build-arg COMMIT=${COMMIT} \ - --build-arg TAG=${TAG} \ - --target build - -cross: ## Cross build for linux, macos and windows - @docker build . \ - --output ./dist \ - --build-arg COMMIT=${COMMIT} \ - --build-arg TAG=${TAG} \ - --target cross - -test: build ## Run tests - @docker build . \ - --build-arg COMMIT=${COMMIT} \ - --build-arg TAG=${TAG} \ - --target test - -e2e: build ## Run tests - go test ./... -v -tags=e2e - -dev: build - @mkdir -p ~/.docker/cli-plugins/ - ln -f -s "${PWD}/dist/docker-ecs" "${HOME}/.docker/cli-plugins/docker-ecs" - -lint: ## Verify Go files - @docker build . --target lint - -fmt: ## Format go files - go fmt ./... - -clean: - rm -rf dist/ - -.PHONY: clean build test dev lint e2e cross fmt diff --git a/ecs/README.md b/ecs/README.md deleted file mode 100644 index 5faeb696..00000000 --- a/ecs/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# Docker CLI plugin for Amazon ECS - -This was announced at AWS Cloud Containers Conference 2020, read the -[blog post](https://www.docker.com/blog/from-docker-straight-to-aws/). - -## Status - -:exclamation: The Docker ECS plugin is still in Beta. -Its design and UX will evolve until 1.0 Final release. - -## Get started - -If you're using macOS or Windows you just need to install -Docker Desktop **Edge** and you will have the ECS integration installed. -You can find links here: -- [macOS](https://hub.docker.com/editions/community/docker-ce-desktop-mac) -- [Windows](https://hub.docker.com/editions/community/docker-ce-desktop-windows) - -Linux install instructions are [here](./docs/get-started-linux.md). - -## Example and documentation - -You can find an application for testing this in [example](./example). - -You can find more documentation about using the Docker ECS integration -[here](https://docs.docker.com/engine/context/ecs-integration/). - -## Architecture - -The Docker ECS integration is a -[Docker CLI plugin](https://docs.docker.com/engine/extend/cli_plugins/) -with the root command of `ecs`. -It requires an AWS profile to select the AWS API credentials from -`~/.aws/credentials` as well as an AWS region. You can read more about CLI AWS -credential management -[here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). -Once setup, the AWS profile and region are stored in a Docker context. - -A `compose.yaml` file is parsed and converted into a -[CloudFormation](https://aws.amazon.com/cloudformation/) template, -which is then used to create all application resources in dependent order. -Resources are cleaned up with the `down` command or in the case of a deployment -failure. - -The architecture of the ECS integration is shown below: - -``` - +--------------------------------------+ - | compose.yaml file | - +--------------------------------------+ -- Load - +--------------------------------------+ - | Compose Model | - +--------------------------------------+ -- Validate - +--------------------------------------+ - | Compose Model suitable for ECS | - +--------------------------------------+ -- Convert - +--------------------------------------+ - | CloudFormation Template | - +--------------------------------------+ -- Apply - +--------------+ +----------------+ - | AWS API | or | stack file | - +--------------+ +----------------+ -``` - -* The _Load_ phase relies on - [compose-go](https://github.com/compose-spec/compose-go). - Any generic code we write for this purpose should be proposed upstream. -* The _Validate_ phase is responsible for injecting sane ECS defaults into the - compose-go model, and validating the `compose.yaml` file does not include - unsupported features. -* The _Convert_ phase produces a CloudFormation template to define all - application resources required to implement the application model on AWS. -* The _Apply_ phase does the actual apply of the CloudFormation template, - either by exporting to a stack file or to deploy on AWS. - -## Application model - -### Services - -Compose services are mapped to ECS services. The Compose specification does not -have support for multi-container services (like Kubernetes pods) or sidecars. -When an ECS feature requires a sidecar, we use a custom Compose extension -(`x-aws-*`) to expose the ECS features as a service-level feature, -and keep the plumbing details from the user. - -### Networking - -We map the "network" abstraction from the Compose model to AWS security groups. -The whole application is created within a single VPC, -security groups are created per Compose network, including the implicit -`default` one. -Services are attached according to the networks declared in Compose model. -This approach means that services attached to a common security group can -communicate together, while services from distinct security groups cannot. -This matches the intent of the Compose network model with the limitation that we -cannot set service aliases per network. - -A [CloudMap](https://aws.amazon.com/cloud-map/) private namespace is created for -each application as `{project}.local`. Services get registered so that we -have service discovery and DNS round-robin -(equivalent to Compose's `endpoint_mode: dnsrr`). Docker images SHOULD include -an entrypoint script to replicate this feature: - -```shell script -if [ ! -z LOCALDOMAIN ]; then echo "search ${LOCALDOMAIN}" >> /etc/resolv.conf; fi -``` diff --git a/ecs/backend.go b/ecs/backend.go index 02cf121a..fd0d4685 100644 --- a/ecs/backend.go +++ b/ecs/backend.go @@ -1,5 +1,3 @@ -// +build ecs - /* Copyright 2020 Docker, Inc. Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,8 +15,8 @@ package ecs import ( "context" - - ecsplugin "github.com/docker/ecs-plugin/pkg/amazon/backend" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" "github.com/docker/api/backend" "github.com/docker/api/compose" @@ -58,19 +56,28 @@ func service(ctx context.Context) (backend.Service, error) { } func getEcsAPIService(ecsCtx store.EcsContext) (*ecsAPIService, error) { - backend, err := ecsplugin.NewBackend(ecsCtx.Profile, ecsCtx.Region) + sess, err := session.NewSessionWithOptions(session.Options{ + Profile: ecsCtx.Profile, + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + Region: aws.String(ecsCtx.Region), + }, + }) if err != nil { return nil, err } + return &ecsAPIService{ - ctx: ecsCtx, - composeBackend: backend, + ctx: ecsCtx, + Region: ecsCtx.Region, + SDK: NewSDK(sess), }, nil } type ecsAPIService struct { - ctx store.EcsContext - composeBackend *ecsplugin.Backend + ctx store.EcsContext + Region string + SDK sdk } func (a *ecsAPIService) ContainerService() containers.Service { @@ -78,7 +85,7 @@ func (a *ecsAPIService) ContainerService() containers.Service { } func (a *ecsAPIService) ComposeService() compose.Service { - return a.composeBackend + return a } func getCloudService() (cloud.Service, error) { diff --git a/ecs/builder.Makefile b/ecs/builder.Makefile deleted file mode 100644 index ae52173d..00000000 --- a/ecs/builder.Makefile +++ /dev/null @@ -1,39 +0,0 @@ -GOOS?=$(shell go env GOOS) -GOARCH?=$(shell go env GOARCH) - -EXTENSION := -ifeq ($(GOOS),windows) - EXTENSION := .exe -endif - -STATIC_FLAGS=CGO_ENABLED=0 -LDFLAGS := "-s -w \ - -X github.com/docker/ecs-plugin/internal.GitCommit=$(COMMIT) \ - -X github.com/docker/ecs-plugin/internal.Version=$(TAG)" -GO_BUILD=$(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS) - -BINARY=dist/docker-ecs -BINARY_WITH_EXTENSION=$(BINARY)$(EXTENSION) - -export DOCKER_BUILDKIT=1 - -all: build - -clean: - rm -rf dist/ - -build: - $(GO_BUILD) -v -o $(BINARY_WITH_EXTENSION) cmd/main/main.go - -cross: - @GOOS=linux GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-linux-amd64 cmd/main/main.go - @GOOS=darwin GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-darwin-amd64 cmd/main/main.go - @GOOS=windows GOARCH=amd64 $(GO_BUILD) -v -o $(BINARY)-windows-amd64.exe cmd/main/main.go - -test: ## Run tests - @$(STATIC_FLAGS) go test -cover $(shell go list ./... | grep -vE 'e2e') - -lint: ## Verify Go files - $(STATIC_FLAGS) golangci-lint run --timeout 10m0s --config ./golangci.yaml ./... - -.PHONY: all clean build cross test dev lint diff --git a/ecs/pkg/amazon/backend/cloudformation.go b/ecs/cloudformation.go similarity index 97% rename from ecs/pkg/amazon/backend/cloudformation.go rename to ecs/cloudformation.go index c6c033db..dda69f00 100644 --- a/ecs/pkg/amazon/backend/cloudformation.go +++ b/ecs/cloudformation.go @@ -1,7 +1,8 @@ -package backend +package ecs import ( "fmt" + "github.com/docker/api/compose" "io/ioutil" "regexp" "strings" @@ -21,7 +22,6 @@ import ( "github.com/compose-spec/compose-go/compatibility" "github.com/compose-spec/compose-go/errdefs" "github.com/compose-spec/compose-go/types" - "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" ) @@ -34,7 +34,7 @@ const ( ) // Convert a compose project into a CloudFormation template -func (b Backend) Convert(project *types.Project) (*cloudformation.Template, error) { +func (b ecsAPIService) Convert(project *types.Project) (*cloudformation.Template, error) { var checker compatibility.Checker = &FargateCompatibilityChecker{ compatibility.AllowList{ Supported: compatibleComposeAttributes, @@ -236,7 +236,7 @@ func (b Backend) Convert(project *types.Project) (*cloudformation.Template, erro func createLogGroup(project *types.Project, template *cloudformation.Template) { retention := 0 - if v, ok := project.Extensions[compose.ExtensionRetention]; ok { + if v, ok := project.Extensions[ExtensionRetention]; ok { retention = v.(int) } logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) @@ -253,11 +253,11 @@ func computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) { return minPercent, maxPercent, nil } updateConfig := service.Deploy.UpdateConfig - min, okMin := updateConfig.Extensions[compose.ExtensionMinPercent] + min, okMin := updateConfig.Extensions[ExtensionMinPercent] if okMin { minPercent = min.(int) } - max, okMax := updateConfig.Extensions[compose.ExtensionMaxPercent] + max, okMax := updateConfig.Extensions[ExtensionMaxPercent] if okMax { maxPercent = max.(int) } @@ -442,7 +442,7 @@ func createTaskExecutionRole(service types.ServiceConfig, err error, definition }) } - if roles, ok := service.Extensions[compose.ExtensionRole]; ok { + if roles, ok := service.Extensions[ExtensionRole]; ok { rolePolicies = append(rolePolicies, iam.Role_Policy{ PolicyDocument: roles, }) @@ -451,7 +451,7 @@ func createTaskExecutionRole(service types.ServiceConfig, err error, definition ECSTaskExecutionPolicy, ECRReadOnlyPolicy, } - if v, ok := service.Extensions[compose.ExtensionManagedPolicies]; ok { + if v, ok := service.Extensions[ExtensionManagedPolicies]; ok { for _, s := range v.([]interface{}) { managedPolicies = append(managedPolicies, s.(string)) } @@ -488,7 +488,7 @@ func createCloudMap(project *types.Project, template *cloudformation.Template) { } func convertNetwork(project *types.Project, net types.NetworkConfig, vpc string, template *cloudformation.Template) string { - if sg, ok := net.Extensions[compose.ExtensionSecurityGroup]; ok { + if sg, ok := net.Extensions[ExtensionSecurityGroup]; ok { logrus.Debugf("Security Group for network %q set by user to %q", net.Name, sg) return sg.(string) } diff --git a/ecs/pkg/amazon/backend/cloudformation_test.go b/ecs/cloudformation_test.go similarity index 99% rename from ecs/pkg/amazon/backend/cloudformation_test.go rename to ecs/cloudformation_test.go index 307b6036..e93dc86b 100644 --- a/ecs/pkg/amazon/backend/cloudformation_test.go +++ b/ecs/cloudformation_test.go @@ -1,4 +1,4 @@ -package backend +package ecs import ( "fmt" @@ -15,7 +15,6 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" - "github.com/docker/ecs-plugin/pkg/compose" "gotest.tools/v3/assert" "gotest.tools/v3/golden" ) diff --git a/ecs/cmd/commands/compose.go b/ecs/cmd/commands/compose.go deleted file mode 100644 index 7fea85e9..00000000 --- a/ecs/cmd/commands/compose.go +++ /dev/null @@ -1,152 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "io" - "os" - "strings" - - "github.com/compose-spec/compose-go/cli" - "github.com/docker/cli/cli/command" - amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" - "github.com/docker/ecs-plugin/pkg/amazon/cloudformation" - "github.com/docker/ecs-plugin/pkg/docker" - "github.com/docker/ecs-plugin/pkg/progress" - "github.com/spf13/cobra" -) - -func ComposeCommand(dockerCli command.Cli) *cobra.Command { - cmd := &cobra.Command{ - Use: "compose", - } - opts := &composeOptions{} - AddFlags(opts, cmd.Flags()) - - cmd.AddCommand( - ConvertCommand(dockerCli, opts), - UpCommand(dockerCli, opts), - DownCommand(dockerCli, opts), - LogsCommand(dockerCli, opts), - PsCommand(dockerCli, opts), - ) - return cmd -} - -type upOptions struct { - loadBalancerArn string -} - -func (o upOptions) LoadBalancerArn() *string { - if o.loadBalancerArn == "" { - return nil - } - return &o.loadBalancerArn -} - -func ConvertCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "convert", - RunE: WithAwsContext(dockerCli, func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error { - opts, err := options.toProjectOptions() - if err != nil { - return err - } - project, err := cli.ProjectFromOptions(opts) - if err != nil { - return err - } - template, err := backend.Convert(project) - if err != nil { - return err - } - json, err := cloudformation.Marshall(template) - if err != nil { - return err - } - fmt.Printf("%s\n", string(json)) - return nil - }), - } - return cmd -} - -func UpCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { - opts := upOptions{} - cmd := &cobra.Command{ - Use: "up", - RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - opts, err := options.toProjectOptions() - if err != nil { - return err - } - - return progress.Run(context.Background(), func(ctx context.Context) error { - return backend.Up(ctx, opts) - }) - }), - } - cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") - return cmd -} - -func PsCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { - opts := upOptions{} - cmd := &cobra.Command{ - Use: "ps", - RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - opts, err := options.toProjectOptions() - if err != nil { - return err - } - status, err := backend.Ps(context.Background(), opts) - if err != nil { - return err - } - printSection(os.Stdout, len(status), func(w io.Writer) { - for _, service := range status { - fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\n", service.ID, service.Name, service.Replicas, service.Desired, strings.Join(service.Ports, ", ")) - } - }, "ID", "NAME", "REPLICAS", "PORTS") - return nil - }), - } - cmd.Flags().StringVar(&opts.loadBalancerArn, "load-balancer", "", "") - return cmd -} - -type downOptions struct { - DeleteCluster bool -} - -func DownCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { - opts := downOptions{} - cmd := &cobra.Command{ - Use: "down", - RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - opts, err := options.toProjectOptions() - if err != nil { - return err - } - return progress.Run(context.Background(), func(ctx context.Context) error { - return backend.Down(ctx, opts) - }) - }), - } - cmd.Flags().BoolVar(&opts.DeleteCluster, "delete-cluster", false, "Delete cluster") - return cmd -} - -func LogsCommand(dockerCli command.Cli, options *composeOptions) *cobra.Command { - cmd := &cobra.Command{ - Use: "logs [PROJECT NAME]", - RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - opts, err := options.toProjectOptions() - if err != nil { - return err - } - return backend.Logs(context.Background(), opts, os.Stdout) - }), - } - return cmd -} diff --git a/ecs/cmd/commands/context.go b/ecs/cmd/commands/context.go deleted file mode 100644 index 080d3b7d..00000000 --- a/ecs/cmd/commands/context.go +++ /dev/null @@ -1,31 +0,0 @@ -package commands - -import ( - "fmt" - - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/docker/cli/cli/command" - amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" - "github.com/docker/ecs-plugin/pkg/docker" - "github.com/spf13/cobra" -) - -type ContextFunc func(ctx docker.AwsContext, backend *amazon.Backend, args []string) error - -func WithAwsContext(dockerCli command.Cli, f ContextFunc) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - ctx, err := docker.GetAwsContext(dockerCli) - if err != nil { - return err - } - backend, err := amazon.NewBackend(ctx.Profile, ctx.Region) - if err != nil { - return err - } - err = f(*ctx, backend, args) - if e, ok := err.(awserr.Error); ok { - return fmt.Errorf(e.Message()) - } - return err - } -} diff --git a/ecs/cmd/commands/opts.go b/ecs/cmd/commands/opts.go deleted file mode 100644 index 2124ca24..00000000 --- a/ecs/cmd/commands/opts.go +++ /dev/null @@ -1,28 +0,0 @@ -package commands - -import ( - "github.com/compose-spec/compose-go/cli" - "github.com/spf13/pflag" -) - -type composeOptions struct { - Name string - WorkingDir string - ConfigPaths []string - Environment []string -} - -func AddFlags(o *composeOptions, flags *pflag.FlagSet) { - flags.StringArrayVarP(&o.ConfigPaths, "file", "f", nil, "Specify an alternate compose file") - flags.StringVarP(&o.Name, "project-name", "n", "", "Specify an alternate project name (default: directory name)") - flags.StringVarP(&o.WorkingDir, "workdir", "w", "", "Working directory") - flags.StringSliceVarP(&o.Environment, "environment", "e", []string{}, "Environment variables") -} - -func (o *composeOptions) toProjectOptions() (*cli.ProjectOptions, error) { - return cli.NewProjectOptions(o.ConfigPaths, - cli.WithOsEnv, - cli.WithEnv(o.Environment), - cli.WithWorkingDirectory(o.WorkingDir), - cli.WithName(o.Name)) -} diff --git a/ecs/cmd/commands/root.go b/ecs/cmd/commands/root.go deleted file mode 100644 index a9c22921..00000000 --- a/ecs/cmd/commands/root.go +++ /dev/null @@ -1,32 +0,0 @@ -package commands - -import ( - "fmt" - - "github.com/docker/cli/cli/command" - "github.com/spf13/cobra" -) - -// NewRootCmd returns the base root command. -func NewRootCmd(dockerCli command.Cli) *cobra.Command { - cmd := &cobra.Command{ - Short: "Docker ECS", - Long: `run multi-container applications on Amazon ECS.`, - Use: "ecs", - Annotations: map[string]string{"experimentalCLI": "true"}, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 0 { - return fmt.Errorf("%q is not a docker ecs command\nSee 'docker ecs --help'", args[0]) - } - cmd.Help() - return nil - }, - } - cmd.AddCommand( - VersionCommand(), - ComposeCommand(dockerCli), - SecretCommand(dockerCli), - SetupCommand(), - ) - return cmd -} diff --git a/ecs/cmd/commands/root_test.go b/ecs/cmd/commands/root_test.go deleted file mode 100644 index 80f2a72d..00000000 --- a/ecs/cmd/commands/root_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package commands - -import ( - "testing" - - "gotest.tools/v3/assert" -) - -func TestUnknownCommand(t *testing.T) { - root := NewRootCmd(nil) - _, _, err := root.Find([]string{"unknown_command"}) - assert.Error(t, err, "unknown command \"unknown_command\" for \"ecs\"") -} diff --git a/ecs/cmd/commands/secret.go b/ecs/cmd/commands/secret.go deleted file mode 100644 index 3246e123..00000000 --- a/ecs/cmd/commands/secret.go +++ /dev/null @@ -1,140 +0,0 @@ -package commands - -import ( - "context" - "errors" - "fmt" - "io" - "os" - "strings" - "text/tabwriter" - - "github.com/docker/cli/cli/command" - amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/pkg/docker" - "github.com/spf13/cobra" -) - -type createSecretOptions struct { - Label string - Username string - Password string - Description string -} - -type deleteSecretOptions struct { - recover bool -} - -func SecretCommand(dockerCli command.Cli) *cobra.Command { - cmd := &cobra.Command{ - Use: "secret", - Short: "Manages secrets", - } - - cmd.AddCommand( - CreateSecret(dockerCli), - InspectSecret(dockerCli), - ListSecrets(dockerCli), - DeleteSecret(dockerCli), - ) - return cmd -} - -func CreateSecret(dockerCli command.Cli) *cobra.Command { - opts := createSecretOptions{} - cmd := &cobra.Command{ - Use: "create NAME", - Short: "Creates a secret.", - RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - if len(args) == 0 { - return errors.New("Missing mandatory parameter: NAME") - } - name := args[0] - - secret := compose.NewSecret(name, opts.Username, opts.Password, opts.Description) - id, err := backend.CreateSecret(context.Background(), secret) - fmt.Println(id) - return err - }), - } - cmd.Flags().StringVarP(&opts.Username, "username", "u", "", "username") - cmd.Flags().StringVarP(&opts.Password, "password", "p", "", "password") - cmd.Flags().StringVarP(&opts.Description, "description", "d", "", "Secret description") - return cmd -} - -func InspectSecret(dockerCli command.Cli) *cobra.Command { - cmd := &cobra.Command{ - Use: "inspect ID", - Short: "Displays secret details", - RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - if len(args) == 0 { - return errors.New("Missing mandatory parameter: ID") - } - id := args[0] - secret, err := backend.InspectSecret(context.Background(), id) - if err != nil { - return err - } - out, err := secret.ToJSON() - if err != nil { - return err - } - fmt.Println(out) - return nil - }), - } - return cmd -} - -func ListSecrets(dockerCli command.Cli) *cobra.Command { - cmd := &cobra.Command{ - Use: "list", - Aliases: []string{"ls"}, - Short: "List secrets stored for the existing account.", - RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - secrets, err := backend.ListSecrets(context.Background()) - if err != nil { - return err - } - - printList(os.Stdout, secrets) - return nil - }), - } - return cmd -} - -func DeleteSecret(dockerCli command.Cli) *cobra.Command { - opts := deleteSecretOptions{} - cmd := &cobra.Command{ - Use: "delete NAME", - Aliases: []string{"rm", "remove"}, - Short: "Removes a secret.", - RunE: WithAwsContext(dockerCli, func(clusteropts docker.AwsContext, backend *amazon.Backend, args []string) error { - if len(args) == 0 { - return errors.New("Missing mandatory parameter: [NAME]") - } - return backend.DeleteSecret(context.Background(), args[0], opts.recover) - }), - } - cmd.Flags().BoolVar(&opts.recover, "recover", false, "Enable recovery.") - return cmd -} - -func printList(out io.Writer, secrets []compose.Secret) { - printSection(out, len(secrets), func(w io.Writer) { - for _, secret := range secrets { - fmt.Fprintf(w, "%s\t%s\t%s\n", secret.ID, secret.Name, secret.Description) - } - }, "ID", "NAME", "DESCRIPTION") -} - -func printSection(out io.Writer, len int, printer func(io.Writer), headers ...string) { - w := tabwriter.NewWriter(out, 20, 1, 3, ' ', 0) - fmt.Fprintln(w, strings.Join(headers, "\t")) - printer(w) - w.Flush() -} diff --git a/ecs/cmd/commands/setup.go b/ecs/cmd/commands/setup.go deleted file mode 100644 index da733f71..00000000 --- a/ecs/cmd/commands/setup.go +++ /dev/null @@ -1,262 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "path/filepath" - "reflect" - "strings" - - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/defaults" - amazon "github.com/docker/ecs-plugin/pkg/amazon/backend" - contextStore "github.com/docker/ecs-plugin/pkg/docker" - "github.com/manifoldco/promptui" - "github.com/spf13/cobra" - "gopkg.in/ini.v1" -) - -const enterLabelPrefix = "Enter " - -type setupOptions struct { - name string - profile string - region string - accessKeyID string - secretAccessKey string -} - -func (s setupOptions) unsetRequiredArgs() []string { - unset := []string{} - if s.profile == "" { - unset = append(unset, "profile") - } - if s.region == "" { - unset = append(unset, "region") - } - return unset -} - -func SetupCommand() *cobra.Command { - var opts setupOptions - - cmd := &cobra.Command{ - Use: "setup", - Short: "", - RunE: func(cmd *cobra.Command, args []string) error { - if requiredFlag := opts.unsetRequiredArgs(); len(requiredFlag) > 0 { - if err := interactiveCli(&opts); err != nil { - return err - } - } - if opts.accessKeyID != "" && opts.secretAccessKey != "" { - if err := saveCredentials(opts.profile, opts.accessKeyID, opts.secretAccessKey); err != nil { - return err - } - } - backend, err := amazon.NewBackend(opts.profile, opts.region) - if err != nil { - return err - } - - context, _, err := backend.CreateContextData(context.Background(), map[string]string{ - amazon.ContextParamProfile: opts.profile, - amazon.ContextParamRegion: opts.region, - }) - if err != nil { - return err - } - return contextStore.NewContext(opts.name, context) - }, - } - cmd.Flags().StringVarP(&opts.name, "name", "n", "ecs", "Context Name") - cmd.Flags().StringVarP(&opts.profile, "profile", "p", "", "AWS Profile") - cmd.Flags().StringVarP(&opts.region, "region", "r", "", "AWS region") - cmd.Flags().StringVarP(&opts.accessKeyID, "aws-key-id", "k", "", "AWS Access Key ID") - cmd.Flags().StringVarP(&opts.secretAccessKey, "aws-secret-key", "s", "", "AWS Secret Access Key") - - return cmd -} - -func interactiveCli(opts *setupOptions) error { - var section ini.Section - - if err := setContextName(opts); err != nil { - return err - } - - section, err := setProfile(opts, section) - if err != nil { - return err - } - - if err := setRegion(opts, section); err != nil { - return err - } - - if err := setCredentials(opts); err != nil { - return err - } - - return nil -} - -func saveCredentials(profile string, accessKeyID string, secretAccessKey string) error { - p := credentials.SharedCredentialsProvider{Profile: profile} - _, err := p.Retrieve() - if err == nil { - fmt.Println("credentials already exists!") - return nil - } - - if err.(awserr.Error).Code() == "SharedCredsLoad" && err.(awserr.Error).Message() == "failed to load shared credentials file" { - err = os.MkdirAll(filepath.Dir(p.Filename), 0700) - if err != nil { - return err - } - _, err = os.Create(p.Filename) - if err != nil { - return err - } - } - - credIni, err := ini.LooseLoad(p.Filename) - if err != nil { - return err - } - section, err := credIni.NewSection(profile) - if err != nil { - return err - } - section.NewKey("aws_access_key_id", accessKeyID) - section.NewKey("aws_secret_access_key", secretAccessKey) - return credIni.SaveTo(p.Filename) -} - -func awsProfiles(filename string) (map[string]ini.Section, error) { - profiles := map[string]ini.Section{"new profile": {}} - if filename == "" { - filename = defaults.SharedConfigFilename() - } - credIni, err := ini.LooseLoad(filename) - if err != nil { - return nil, err - } - if err != nil { - return nil, err - } - for _, section := range credIni.Sections() { - if strings.HasPrefix(section.Name(), "profile") { - profiles[section.Name()[len("profile "):]] = *section - } - } - return profiles, nil -} - -func setContextName(opts *setupOptions) error { - if opts.name == "ecs" { - result, err := promptString(opts.name, "context name", enterLabelPrefix, 2) - if err != nil { - return err - } - opts.name = result - } - return nil -} - -func setProfile(opts *setupOptions, section ini.Section) (ini.Section, error) { - profilesList, err := awsProfiles("") - if err != nil { - return ini.Section{}, err - } - section, ok := profilesList[opts.profile] - if !ok { - prompt := promptui.Select{ - Label: "Select AWS Profile", - Items: reflect.ValueOf(profilesList).MapKeys(), - } - _, result, err := prompt.Run() - if result == "new profile" { - result, err := promptString(opts.profile, "profile name", enterLabelPrefix, 2) - if err != nil { - return ini.Section{}, err - } - opts.profile = result - } else { - section = profilesList[result] - opts.profile = result - } - if err != nil { - return ini.Section{}, err - } - } - return section, nil -} - -func setRegion(opts *setupOptions, section ini.Section) error { - defaultRegion := opts.region - if defaultRegion == "" && section.Name() != "" { - region, err := section.GetKey("region") - if err == nil { - defaultRegion = region.Value() - } - } - result, err := promptString(defaultRegion, "region", enterLabelPrefix, 2) - if err != nil { - return err - } - opts.region = result - return nil -} - -func setCredentials(opts *setupOptions) error { - prompt := promptui.Prompt{ - Label: "Enter credentials", - IsConfirm: true, - } - _, err := prompt.Run() - if err == nil { - result, err := promptString(opts.accessKeyID, "AWS Access Key ID", enterLabelPrefix, 3) - if err != nil { - return err - } - opts.accessKeyID = result - - prompt = promptui.Prompt{ - Label: "Enter AWS Secret Access Key", - Validate: validateMinLen("AWS Secret Access Key", 3), - Mask: '*', - Default: opts.secretAccessKey, - } - result, err = prompt.Run() - if err != nil { - return err - } - opts.secretAccessKey = result - } - return nil -} - -func promptString(defaultValue string, label string, labelPrefix string, minLength int) (string, error) { - prompt := promptui.Prompt{ - Label: labelPrefix + label, - Validate: validateMinLen(label, minLength), - Default: defaultValue, - } - result, err := prompt.Run() - if err != nil { - return "", err - } - return result, nil -} - -func validateMinLen(label string, minLength int) func(input string) error { - return func(input string) error { - if len(input) < minLength { - return fmt.Errorf("%s must have more than %d characters", label, minLength) - } - return nil - } -} diff --git a/ecs/cmd/commands/version.go b/ecs/cmd/commands/version.go deleted file mode 100644 index ce025389..00000000 --- a/ecs/cmd/commands/version.go +++ /dev/null @@ -1,20 +0,0 @@ -package commands - -import ( - "fmt" - - "github.com/docker/ecs-plugin/internal" - - "github.com/spf13/cobra" -) - -func VersionCommand() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Show version.", - RunE: func(cmd *cobra.Command, args []string) error { - fmt.Fprintf(cmd.OutOrStdout(), "Docker ECS plugin %s (%s)\n", internal.Version, internal.GitCommit) - return nil - }, - } -} diff --git a/ecs/cmd/commands/version_test.go b/ecs/cmd/commands/version_test.go deleted file mode 100644 index 43a9ab81..00000000 --- a/ecs/cmd/commands/version_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package commands - -import ( - "bytes" - "strings" - "testing" - - "github.com/docker/ecs-plugin/internal" - - "gotest.tools/v3/assert" -) - -func TestVersion(t *testing.T) { - root := NewRootCmd(nil) - var out bytes.Buffer - root.SetOut(&out) - root.SetArgs([]string{"version"}) - root.Execute() - assert.Check(t, strings.Contains(out.String(), internal.Version)) -} diff --git a/ecs/cmd/main/main.go b/ecs/cmd/main/main.go deleted file mode 100644 index f4140394..00000000 --- a/ecs/cmd/main/main.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "github.com/docker/ecs-plugin/cmd/commands" - "github.com/docker/ecs-plugin/internal" - - "github.com/docker/cli/cli-plugins/manager" - "github.com/docker/cli/cli-plugins/plugin" - "github.com/docker/cli/cli/command" - "github.com/spf13/cobra" -) - -func main() { - plugin.Run(func(dockerCli command.Cli) *cobra.Command { - cmd := commands.NewRootCmd(dockerCli) - return cmd - }, manager.Metadata{ - SchemaVersion: "0.1.0", - Vendor: "Docker Inc.", - Version: internal.Version, - Experimental: true, - }) -} diff --git a/ecs/pkg/console/colors.go b/ecs/colors.go similarity index 98% rename from ecs/pkg/console/colors.go rename to ecs/colors.go index 672b61f5..3af58af5 100644 --- a/ecs/pkg/console/colors.go +++ b/ecs/colors.go @@ -1,4 +1,4 @@ -package console +package ecs import ( "fmt" diff --git a/ecs/pkg/amazon/backend/compatibility.go b/ecs/compatibility.go similarity index 99% rename from ecs/pkg/amazon/backend/compatibility.go rename to ecs/compatibility.go index d217ca48..2fdd527a 100644 --- a/ecs/pkg/amazon/backend/compatibility.go +++ b/ecs/compatibility.go @@ -1,4 +1,4 @@ -package backend +package ecs import ( "github.com/compose-spec/compose-go/compatibility" diff --git a/ecs/context.go b/ecs/context.go index de7a3415..46c2422e 100644 --- a/ecs/context.go +++ b/ecs/context.go @@ -1,5 +1,3 @@ -// +build ecs - /* Copyright 2020 Docker, Inc. Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/ecs/pkg/amazon/backend/convert.go b/ecs/convert.go similarity index 98% rename from ecs/pkg/amazon/backend/convert.go rename to ecs/convert.go index 2056dad4..62eca65a 100644 --- a/ecs/pkg/amazon/backend/convert.go +++ b/ecs/convert.go @@ -1,4 +1,4 @@ -package backend +package ecs import ( "encoding/json" @@ -16,9 +16,8 @@ import ( "github.com/awslabs/goformation/v4/cloudformation/ecs" "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/compose-spec/compose-go/types" + "github.com/docker/api/ecs/secrets" "github.com/docker/cli/opts" - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/secrets" "github.com/joho/godotenv" ) @@ -81,7 +80,7 @@ func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi ValueFrom: secretConfig.Name, }) var keys []string - if ext, ok := secretConfig.Extensions[compose.ExtensionKeys]; ok { + if ext, ok := secretConfig.Extensions[ExtensionKeys]; ok { if key, ok := ext.(string); ok { keys = append(keys, key) } else { @@ -456,7 +455,7 @@ func toHostEntryPtr(hosts types.HostsList) []ecs.TaskDefinition_HostEntry { func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name for key, value := range service.Extensions { - if key == compose.ExtensionPullCredentials { + if key == ExtensionPullCredentials { return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: value.(string)} } } diff --git a/ecs/doc.go b/ecs/doc.go deleted file mode 100644 index 2ea49a88..00000000 --- a/ecs/doc.go +++ /dev/null @@ -1,14 +0,0 @@ -/* - Copyright 2020 Docker, Inc. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package ecs diff --git a/ecs/docs/get-started-linux.md b/ecs/docs/get-started-linux.md deleted file mode 100644 index afada890..00000000 --- a/ecs/docs/get-started-linux.md +++ /dev/null @@ -1,82 +0,0 @@ -Getting Started with Docker AWS ECS Plugin Beta on Linux --------------------------------------------------------- - -The beta release of [AWS ECS](https://aws.amazon.com/ecs/) support for the -Docker CLI is shipped as a CLI plugin. Later releases will be included as part -of the Docker CLI. - -This plugin is included as part of Docker Desktop on Windows and macOS but on -Linux it needs to be installed manually. - -## Prerequisites - -* [Docker 19.03 or later](https://docs.docker.com/get-docker/) - -## Step by step install - -### Download - -You can download the Docker ECS plugin from this repository using the following -command: - -```console -$ curl -LO https://github.com/docker/ecs-plugin/releases/latest/download/docker-ecs-linux-amd64 -``` - -You will then need to make it executable: - -```console -$ chmod +x docker-ecs-linux-amd64 -``` - -### Plugin install - -In order for the Docker CLI to use the downloaded plugin, you will need to move -it to the right place: - -```console -$ mkdir -p /usr/local/lib/docker/cli-plugins - -$ mv docker-ecs-linux-amd64 /usr/local/lib/docker/cli-plugins/docker-ecs -``` - -You can put the CLI plugin into any of the following directories: - -* `/usr/local/lib/docker/cli-plugins` -* `/usr/local/libexec/docker/cli-plugins` -* `/usr/lib/docker/cli-plugins` -* `/usr/libexec/docker/cli-plugins` - -Finally you need to enable the experimental features on the CLI. This can be -done by setting the environment variable `DOCKER_CLI_EXPERIMENTAL=enabled` or by -setting `experimental` to `"enabled"` in your Docker config found at -`~/.docker/config.json`: - -```console -$ export DOCKER_CLI_EXPERIMENTAL=enabled - -$ DOCKER_CLI_EXPERIMENTAL=enabled docker help - -$ cat ~/.docker/config.json -{ - "experimental" : "enabled", - "auths" : { - "https://index.docker.io/v1/" : { - - } - } -} -``` - -To verify the CLI plugin installation, you can check that it appears in the CLI -help output or by outputting the plugin version: - -```console -$ docker help | grep ecs - ecs* Docker ECS (Docker Inc., 0.0.1) - -$ docker ecs version -Docker ECS plugin 0.0.1 -``` - -You are now ready to [start deploying to ECS](https://docs.docker.com/engine/context/ecs-integration/) diff --git a/ecs/docs/requirements.md b/ecs/docs/requirements.md deleted file mode 100644 index d142de4f..00000000 --- a/ecs/docs/requirements.md +++ /dev/null @@ -1,31 +0,0 @@ -## Requirements - -This plugin relies on AWS API credentials, using the same configuration files as -the AWS command line. - -Such credentials can be configured by the `docker ecs setup` command, either by -selecting an existing AWS CLI profile from existing config files, or by creating -one passing an AWS access key ID and secret access key. - -## Permissions - -AWS accounts (or IAM roles) used with the ECS plugin require following permissions: - -- ec2:DescribeSubnets -- ec2:DescribeVpcs -- iam:CreateServiceLinkedRole -- iam:AttachRolePolicy -- cloudformation:* -- ecs:* -- logs:* -- servicediscovery:* -- elasticloadbalancing:* - - -## Okta support - -For those relying on [aws-okta](https://github.com/segmentio/aws-okta) to access a managed AWS account -(as we do at Docker), you can populate your aws config files with temporary access tokens using: -```shell script -aws-okta write-to-credentials ~/.aws/credentials -``` diff --git a/ecs/pkg/amazon/backend/down.go b/ecs/down.go similarity index 52% rename from ecs/pkg/amazon/backend/down.go rename to ecs/down.go index 0d235ad7..849b1c67 100644 --- a/ecs/pkg/amazon/backend/down.go +++ b/ecs/down.go @@ -1,26 +1,25 @@ -package backend +package ecs import ( "context" "github.com/compose-spec/compose-go/cli" - "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) Down(ctx context.Context, options *cli.ProjectOptions) error { +func (b *ecsAPIService) Down(ctx context.Context, options *cli.ProjectOptions) error { name, err := b.projectName(options) if err != nil { return err } - err = b.api.DeleteStack(ctx, name) + err = b.SDK.DeleteStack(ctx, name) if err != nil { return err } - return b.WaitStackCompletion(ctx, name, compose.StackDelete) + return b.WaitStackCompletion(ctx, name, StackDelete) } -func (b *Backend) projectName(options *cli.ProjectOptions) (string, error) { +func (b *ecsAPIService) projectName(options *cli.ProjectOptions) (string, error) { name := options.Name if name == "" { project, err := cli.ProjectFromOptions(options) diff --git a/ecs/example/Makefile b/ecs/example/Makefile deleted file mode 100644 index deba2b83..00000000 --- a/ecs/example/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -REPO_NAMESPACE ?= ${USER} -FRONTEND_IMG = ${REPO_NAMESPACE}/timestamper -REGISTRY_ID ?= PUT_ECR_REGISTRY_ID_HERE -DOCKER_PUSH_REPOSITORY=dkr.ecr.us-west-2.amazonaws.com - -all: build-image - -create-ecr: - aws ecr create-repository --repository-name ${FRONTEND_IMG} - -build-image: - docker build -t $(REGISTRY_ID).$(DOCKER_PUSH_REPOSITORY)/$(FRONTEND_IMG) ./app - docker build -t $(FRONTEND_IMG) ./app - -push-image-ecr: - aws ecr get-login-password --region us-west-2 | docker login -u AWS --password-stdin $(REGISTRY_ID).$(DOCKER_PUSH_REPOSITORY) - docker push $(REGISTRY_ID).$(DOCKER_PUSH_REPOSITORY)/$(FRONTEND_IMG) - -push-image-hub: - docker push $(FRONTEND_IMG) - -clean: - @docker context use default - @docker context rm aws || true - @docker-compose rm -f || true diff --git a/ecs/example/README.md b/ecs/example/README.md deleted file mode 100644 index e8dbbd73..00000000 --- a/ecs/example/README.md +++ /dev/null @@ -1,181 +0,0 @@ -## Compose sample application - -This sample application was demoed as part of the AWS Cloud Containers -Conference on 2020-07-09. It has been tested on Linux and macOS. - -Note that `$` is used to denote commands in blocks where the command and its -output are included. - -### Python/Flask application - -``` -+--------------------+ +------------------+ -| | | | -| Python Flask | timestamps | Redis | -| Application |------------->| | -| | | | -+--------------------+ +------------------+ -``` - -### Things you'll need to do - -There are a number of places you'll need to fill in information. -You can find them with: - -```console -$ grep -r '<<<' ./* -./docker-compose.yml: x-aws-pull_credentials: <<>> -./docker-compose.yml: image: <<>>/timestamper -## Walk through -``` - -### Setup pull credentials for private Docker Hub repositories - -You should use a Personal Access Token (PAT) rather than your account password. -If you have 2FA enabled on your Hub account you will need to create a PAT. -You can read more about managing access tokens here: -https://docs.docker.com/docker-hub/access-tokens/ - - -You can then create `DockerHubToken` secret on [AWS Secret Manager](https://aws.amazon.com/secrets-manager/) using following command - -```console -docker ecs secret create -d MyKey -u myhubusername -p myhubpat DockerHubToken -``` - -### Create an AWS Docker context and list available contexts - -To initialize the Docker ECS integration, you will need to run the `setup` -command. This will create a Docker context that works with AWS ECS. - -```console -$ docker ecs setup -Enter context name: aws -✔ sandbox.devtools.developer -Enter cluster name: -Enter region: us-west-2 -✗ Enter credentials: -``` - -You can verify that the context was created by listing your Docker contexts: - -```console -$ docker context ls -NAME DESCRIPTION DOCKER ENDPOINT KUBERNETES ENDPOINT ORCHESTRATOR -aws -default * Current DOCKER_HOST based configuration unix:///var/run/docker.sock swarm -``` - -### Test locally - -The first step is to test your application works locally. To do this, you will -need to switch to using the default local context so that you are targeting your -local machine. - -```console -docker context use default -``` - -You can then run the application using `docker-compose`: - -```console -docker-compose up -``` - -Once the application has started, you can navigate to http://localhost:5000 -using your Web browser using the following command: - -```console -open http://localhost:5000 -``` - -### Push images to Docker Hub for ECS (ECS cannot see your local image cache) - -In order to run your application in the cloud, you will need your container -images to be in a registry. You can push them from your local machine using: - -```console -docker-compose push -``` - -You can verify that this command pushed to the Docker Hub by -[logging in](https://hub.docker.com) and looking for the `timestamper` -repository under your user name. - -### Switch to ECS context and launch the app - -Now that you've tested the application works locally and that you've pushed the -container images to the Docker Hub, you can switch to using the `aws` context -you created earlier. - -```console -docker context use aws -``` - -Running the application on ECS is then as simple as doing a `compose up`: - -```console -docker ecs compose up -``` - -### Check out the CLI - -Once the application is running in ECS, you can list the running containers with -the `ps` command. Note that you will need to run this from the directory where -you Compose file is. - -```console -docker ecs compose ps -``` - -You can also read the application logs using `compose logs`: - -```console -docker ecs compose logs -``` - -### Check out the AWS console - -You can see all the AWS components created for your running application in the -AWS console. There you will find: - -- CloudFormation being used to manage all the infrastructure -- CloudWatch for logs -- Security Groups for network policies -- Load balancers (ELB for this example / ALB if your app only uses 80/443) - -### Checkout CloudFormation - -The ECS Docker CLI integration has the ability to output the CloudFormation -template used to create the application in the `compose convert` command. You -can see this by running: - -```console -docker ecs compose convert -``` - -### Stop the meters - -To shut down your application, you simply need to run: - -```console -docker ecs compose down -``` - -## Using Amazon ECR instead of Docker Hub - -If you'd like to use AWS ECR instead of Docker Hub, the [Makefile](Makefile) has -an example setup for creating an ECR repository and pushing to it. -You'll need to have the AWS CLI installed and your AWS credentials available. - -```console -make create-ecr -REGISTRY_ID= make build-image -REGISTRY_ID= make push-image-ecr -``` - -Note that you will need to change the name of the image in the -[Compose file](docker-compose.yml). - -If you want to use this often, you'll likely want to replace -`PUT_ECR_REGISTRY_ID_HERE` with the value from above. diff --git a/ecs/example/app/Dockerfile b/ecs/example/app/Dockerfile deleted file mode 100644 index 64469d28..00000000 --- a/ecs/example/app/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:3.7-alpine -WORKDIR /app -COPY requirements.txt /app -RUN pip3 install -r requirements.txt -COPY . /app -ENTRYPOINT ["/app/scripts/entrypoint.sh"] -CMD ["python3", "app.py"] diff --git a/ecs/example/app/app.py b/ecs/example/app/app.py deleted file mode 100644 index e54b19a2..00000000 --- a/ecs/example/app/app.py +++ /dev/null @@ -1,19 +0,0 @@ - -from flask import Flask -from flask import render_template -from redis import StrictRedis -from datetime import datetime - -app = Flask(__name__) -redis = StrictRedis(host='backend', port=6379) - - -@app.route('/') -def home(): - redis.lpush('times', datetime.now().strftime('%Y-%m-%dT%H:%M:%S%z')) - return render_template('index.html', title='Home', - times=redis.lrange('times', 0, -1)) - - -if __name__ == '__main__': - app.run(host='0.0.0.0', debug=True) diff --git a/ecs/example/app/requirements.txt b/ecs/example/app/requirements.txt deleted file mode 100644 index 1a5dc97b..00000000 --- a/ecs/example/app/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -flask -redis diff --git a/ecs/example/app/scripts/entrypoint.sh b/ecs/example/app/scripts/entrypoint.sh deleted file mode 100755 index 19866024..00000000 --- a/ecs/example/app/scripts/entrypoint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh - -if [ "${LOCALDOMAIN}" != "" ]; then echo "search ${LOCALDOMAIN}" >> /etc/resolv.conf; fi -exec "$@" diff --git a/ecs/example/app/templates/index.html b/ecs/example/app/templates/index.html deleted file mode 100644 index 91efdaef..00000000 --- a/ecs/example/app/templates/index.html +++ /dev/null @@ -1,125 +0,0 @@ - - - - - - - - - - - Hello, Docker! - - -
- - -
-

Hello, Docker Folks!

-
- -
-
- - - Timestamp! - -
- -
- -
- - - - - - - - {% for t in times %} - - - - {% endfor %} - -
Timestamp
{{ t.decode('utf-8') }}
-
- -
- - - - - - - - diff --git a/ecs/example/docker-compose.yml b/ecs/example/docker-compose.yml deleted file mode 100644 index 9d08d45d..00000000 --- a/ecs/example/docker-compose.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3.8" -services: - frontend: - build: app - x-aws-pull_credentials: <<>> - image: <<>>/timestamper - ports: - - "5000:5000" - depends_on: - - backend - backend: - image: redis:alpine diff --git a/ecs/go.mod b/ecs/go.mod deleted file mode 100644 index b6a1f1d2..00000000 --- a/ecs/go.mod +++ /dev/null @@ -1,61 +0,0 @@ -module github.com/docker/ecs-plugin - -require ( - github.com/Microsoft/hcsshim v0.8.7 // indirect - github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d // indirect - github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect - github.com/aws/aws-sdk-go v1.33.18 - github.com/awslabs/goformation/v4 v4.14.0 - github.com/bitly/go-hostpool v0.1.0 // indirect - github.com/bitly/go-simplejson v0.5.0 // indirect - github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect - github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 - github.com/bugsnag/bugsnag-go v1.5.3 // indirect - github.com/bugsnag/panicwrap v1.2.0 // indirect - github.com/cenkalti/backoff v2.2.1+incompatible // indirect - github.com/cloudflare/cfssl v1.4.1 // indirect - github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457 - github.com/containerd/console v1.0.0 - github.com/containerd/containerd v1.3.2 // indirect - github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb // indirect - github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 - github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/docker/docker v1.4.2-0.20200128034134-2ebaeef943cc // indirect - github.com/docker/docker-credential-helpers v0.6.3 // indirect - github.com/docker/go v1.5.1-1 // indirect - github.com/docker/go-metrics v0.0.1 // indirect - github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect - github.com/gofrs/uuid v3.2.0+incompatible // indirect - github.com/gogo/protobuf v1.3.1 // indirect - github.com/gorilla/mux v1.7.3 // indirect - github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/jinzhu/gorm v1.9.12 // indirect - github.com/joho/godotenv v1.3.0 - github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect - github.com/lib/pq v1.3.0 // indirect - github.com/manifoldco/promptui v0.7.0 - github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect - github.com/miekg/pkcs11 v1.0.3 // indirect - github.com/mitchellh/mapstructure v1.3.3 - github.com/moby/term v0.0.0-20200611042045-63b9a826fb74 - github.com/morikuni/aec v1.0.0 - github.com/onsi/ginkgo v1.11.0 // indirect - github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/sirupsen/logrus v1.6.0 - github.com/smartystreets/goconvey v1.6.4 // indirect - github.com/spf13/cobra v0.0.5 - github.com/spf13/pflag v1.0.5 - github.com/theupdateframework/notary v0.6.1 // indirect - github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect - golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect - google.golang.org/grpc v1.27.0 // indirect - gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect - gopkg.in/fatih/pool.v2 v2.0.0 // indirect - gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect - gopkg.in/ini.v1 v1.55.0 - gotest.tools/v3 v3.0.2 - vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect -) - -go 1.14 diff --git a/ecs/go.sum b/ecs/go.sum deleted file mode 100644 index 168b5633..00000000 --- a/ecs/go.sum +++ /dev/null @@ -1,496 +0,0 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c/go.mod h1:hSVuE3qU7grINVSwrmzHfpg9k87ALBk+XaualNyUzI4= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= -github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= -github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= -github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.33.18 h1:Ccy1SV2SsgJU3rfrD+SOhQ0jvuzfrFuja/oKI86ruPw= -github.com/aws/aws-sdk-go v1.33.18/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/awslabs/goformation/v4 v4.14.0 h1:E2Pet9eIqA4qzt3dzzzE4YN83V4Kyfbcio0VokBC9TA= -github.com/awslabs/goformation/v4 v4.14.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= -github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= -github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 h1:gfAMKE626QEuKG3si0pdTRcr/YEbBoxY+3GOH3gWvl4= -github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129/go.mod h1:u9UyCz2eTrSGy6fbupqJ54eY5c4IC8gREQ1053dK12U= -github.com/bugsnag/bugsnag-go v1.5.3 h1:yeRUT3mUE13jL1tGwvoQsKdVbAsQx9AJ+fqahKveP04= -github.com/bugsnag/bugsnag-go v1.5.3/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA= -github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= -github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiKw= -github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo= -github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= -github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= -github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457 h1:8ely1LF7H02sIWz6QjgU53YBCiRpYlM9F9u1MeE1ZPk= -github.com/compose-spec/compose-go v0.0.0-20200811091145-837f8f4de457/go.mod h1:cS0vAvM6u9yjJgKWIH2yiqYMWO7WGJb+c0Irw+RefqU= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1 h1:uict5mhHFTzKLUCufdSLym7z/J0CbBJT59lYbP9wtbg= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v1.0.0 h1:fU3UuQapBs+zLJu82NhR11Rif1ny2zfMMAyPJzSN5tQ= -github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2 h1:ForxmXkA6tPIvffbrDAcPUIB32QgXkt2XFj+F0UxetA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb h1:nXPkFq8X1a9ycY3GYQpFNxHh3j2JgY7zDZfq2EXMIzk= -github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492 h1:FwssHbCDJD025h+BchanCwE1Q8fyMgqDr2mOQAWOLGw= -github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.4.2-0.20200128034134-2ebaeef943cc h1:2xtQXEoAs2hjCs4Ez4/KT2mDaYrXwcUi7TCkfyT+n2k= -github.com/docker/docker v1.4.2-0.20200128034134-2ebaeef943cc/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go v1.5.1-1 h1:hr4w35acWBPhGBXlzPoHpmZ/ygPjnmFVxGxxGnMyP7k= -github.com/docker/go v1.5.1-1/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= -github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE= -github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q= -github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= -github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/kisom/goutils v1.1.0/go.mod h1:+UBTfd78habUYWFbNWTJNG+jNG/i/lGURakr4A/yNRw= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c= -github.com/lib/pq v0.0.0-20180201184707-88edab080323/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= -github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/manifoldco/promptui v0.7.0 h1:3l11YT8tm9MnwGFQ4kETwkzpAwY2Jt9lCrumCUW4+z4= -github.com/manifoldco/promptui v0.7.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= -github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= -github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mjibson/esc v0.2.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= -github.com/moby/term v0.0.0-20200611042045-63b9a826fb74 h1:kvRIeqJNICemq2UFLx8q/Pj+1IRNZS0XPTaMFkuNsvg= -github.com/moby/term v0.0.0-20200611042045-63b9a826fb74/go.mod h1:pJ0Ot5YGdTcMdxnPMyGCfAr6fKXe0g9cDlz16MuFEBE= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= -github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.2.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b h1:jUK33OXuZP/l6babJtnLo1qsGvq6G9so9KMflGAm4YA= -github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b/go.mod h1:8458kAagoME2+LN5//WxE71ysZ3B7r22fdgb7qVmXSY= -github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522 h1:fOCp11H0yuyAt2wqlbJtbyPzSgaxHTv8uN1pMpkG1t8= -github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522/go.mod h1:tQTYKOQgxoH3v6dEmdHiz4JG+nbxWwM5fgPQUpSZqVQ= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/theupdateframework/notary v0.6.1 h1:7wshjstgS9x9F5LuB1L5mBI2xNMObWqjz+cjWoom6l0= -github.com/theupdateframework/notary v0.6.1/go.mod h1:MOfgIfmox8s7/7fduvB2xyPPMJCrjRLRizA8OFwpnKY= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg2K1HrLU= -github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56 h1:yhqBHs09SmmUoNOHc9jgK4a60T3XFRtPAkYxVnqgY50= -github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= -github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20190729165852-9051775e6a2e h1:mvOa4+/DXStR4ZXOks/UsjeFdn5O5JpLUtzqk9U8xXw= -github.com/zmap/zcrypto v0.0.0-20190729165852-9051775e6a2e/go.mod h1:w7kd3qXHh8FNaczNjslXqvFQiv5mMWRXlL9klTUAHc8= -github.com/zmap/zlint v0.0.0-20190806154020-fd021b4cfbeb h1:vxqkjztXSaPVDc8FQCdHTaejm2x747f6yPbnu1h2xkg= -github.com/zmap/zlint v0.0.0-20190806154020-fd021b4cfbeb/go.mod h1:29UiAJNsiVdvTBFCJW8e3q6dcDbOoPkhMgttOSCIMMY= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM= -golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYRuq8JQ1aa7LJt8EXVyo= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/dancannon/gorethink.v3 v3.0.5 h1:/g7PWP7zUS6vSNmHSDbjCHQh1Rqn8Jy6zSMQxAsBSMQ= -gopkg.in/dancannon/gorethink.v3 v3.0.5/go.mod h1:GXsi1e3N2OcKhcP6nsYABTiUejbWMFO4GY5a4pEaeEc= -gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= -gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y2O4MU= -gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= -gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ= -gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/kubernetes v1.13.0 h1:qTfB+u5M92k2fCCCVP2iuhgwwSOv1EkAkvQY1tQODD8= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= -vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/ecs/golangci.yaml b/ecs/golangci.yaml deleted file mode 100644 index 39629336..00000000 --- a/ecs/golangci.yaml +++ /dev/null @@ -1,12 +0,0 @@ -run: - deadline: 2m - -linters: - disable-all: true - enable: - - gofmt - - goimports - - golint - - gosimple - - ineffassign - - misspell \ No newline at end of file diff --git a/ecs/pkg/amazon/backend/iam.go b/ecs/iam.go similarity index 98% rename from ecs/pkg/amazon/backend/iam.go rename to ecs/iam.go index 4b282020..3dca431b 100644 --- a/ecs/pkg/amazon/backend/iam.go +++ b/ecs/iam.go @@ -1,4 +1,4 @@ -package backend +package ecs const ( ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" diff --git a/ecs/internal/version.go b/ecs/internal/version.go deleted file mode 100644 index b0fc1f6a..00000000 --- a/ecs/internal/version.go +++ /dev/null @@ -1,8 +0,0 @@ -package internal - -var ( - // Version is the git tag that this was built from. - Version = "unknown" - // GitCommit is the commit that this was built from. - GitCommit = "unknown" -) diff --git a/ecs/pkg/amazon/backend/list.go b/ecs/list.go similarity index 71% rename from ecs/pkg/amazon/backend/list.go rename to ecs/list.go index 59150314..7642471b 100644 --- a/ecs/pkg/amazon/backend/list.go +++ b/ecs/list.go @@ -1,26 +1,26 @@ -package backend +package ecs import ( "context" "fmt" + "github.com/docker/api/compose" "strings" "github.com/compose-spec/compose-go/cli" - "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) Ps(ctx context.Context, options *cli.ProjectOptions) ([]compose.ServiceStatus, error) { +func (b *ecsAPIService) Ps(ctx context.Context, options *cli.ProjectOptions) ([]compose.ServiceStatus, error) { projectName, err := b.projectName(options) if err != nil { return nil, err } - parameters, err := b.api.ListStackParameters(ctx, projectName) + parameters, err := b.SDK.ListStackParameters(ctx, projectName) if err != nil { return nil, err } cluster := parameters[ParameterClusterName] - resources, err := b.api.ListStackResources(ctx, projectName) + resources, err := b.SDK.ListStackResources(ctx, projectName) if err != nil { return nil, err } @@ -37,7 +37,7 @@ func (b *Backend) Ps(ctx context.Context, options *cli.ProjectOptions) ([]compos if len(servicesARN) == 0 { return nil, nil } - status, err := b.api.DescribeServices(ctx, cluster, servicesARN) + status, err := b.SDK.DescribeServices(ctx, cluster, servicesARN) if err != nil { return nil, err } diff --git a/ecs/pkg/amazon/backend/logs.go b/ecs/logs.go similarity index 77% rename from ecs/pkg/amazon/backend/logs.go rename to ecs/logs.go index fac65ba9..ec36985f 100644 --- a/ecs/pkg/amazon/backend/logs.go +++ b/ecs/logs.go @@ -1,4 +1,4 @@ -package backend +package ecs import ( "bytes" @@ -11,11 +11,9 @@ import ( "strings" "github.com/compose-spec/compose-go/cli" - - "github.com/docker/ecs-plugin/pkg/console" ) -func (b *Backend) Logs(ctx context.Context, options *cli.ProjectOptions, writer io.Writer) error { +func (b *ecsAPIService) Logs(ctx context.Context, options *cli.ProjectOptions, writer io.Writer) error { name := options.Name if name == "" { project, err := cli.ProjectFromOptions(options) @@ -25,8 +23,8 @@ func (b *Backend) Logs(ctx context.Context, options *cli.ProjectOptions, writer name = project.Name } - err := b.api.GetLogs(ctx, name, &logConsumer{ - colors: map[string]console.ColorFunc{}, + err := b.SDK.GetLogs(ctx, name, &logConsumer{ + colors: map[string]ColorFunc{}, width: 0, writer: writer, }) @@ -43,7 +41,7 @@ func (b *Backend) Logs(ctx context.Context, options *cli.ProjectOptions, writer func (l *logConsumer) Log(service, container, message string) { cf, ok := l.colors[service] if !ok { - cf = <-console.Rainbow + cf = <-Rainbow l.colors[service] = cf l.computeWidth() } @@ -66,7 +64,7 @@ func (l *logConsumer) computeWidth() { } type logConsumer struct { - colors map[string]console.ColorFunc + colors map[string]ColorFunc width int writer io.Writer } diff --git a/ecs/pkg/amazon/cloudformation/marshall.go b/ecs/marshall.go similarity index 98% rename from ecs/pkg/amazon/cloudformation/marshall.go rename to ecs/marshall.go index 034bf809..41456362 100644 --- a/ecs/pkg/amazon/cloudformation/marshall.go +++ b/ecs/marshall.go @@ -1,4 +1,4 @@ -package cloudformation +package ecs import ( "encoding/json" diff --git a/ecs/pkg/amazon/amazon.go b/ecs/pkg/amazon/amazon.go deleted file mode 100644 index f6c588d1..00000000 --- a/ecs/pkg/amazon/amazon.go +++ /dev/null @@ -1,8 +0,0 @@ -package amazon - -import ( - "github.com/docker/ecs-plugin/pkg/amazon/backend" - "github.com/docker/ecs-plugin/pkg/compose" -) - -var _ compose.API = &backend.Backend{} diff --git a/ecs/pkg/amazon/backend/backend.go b/ecs/pkg/amazon/backend/backend.go deleted file mode 100644 index 2b265f3e..00000000 --- a/ecs/pkg/amazon/backend/backend.go +++ /dev/null @@ -1,30 +0,0 @@ -package backend - -import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/docker/ecs-plugin/pkg/amazon/sdk" -) - -func NewBackend(profile string, region string) (*Backend, error) { - sess, err := session.NewSessionWithOptions(session.Options{ - Profile: profile, - SharedConfigState: session.SharedConfigEnable, - Config: aws.Config{ - Region: aws.String(region), - }, - }) - if err != nil { - return nil, err - } - - return &Backend{ - Region: region, - api: sdk.NewAPI(sess), - }, nil -} - -type Backend struct { - Region string - api sdk.API -} diff --git a/ecs/pkg/amazon/backend/context.go b/ecs/pkg/amazon/backend/context.go deleted file mode 100644 index c1865117..00000000 --- a/ecs/pkg/amazon/backend/context.go +++ /dev/null @@ -1,33 +0,0 @@ -package backend - -import ( - "context" - "fmt" - - "github.com/docker/ecs-plugin/pkg/docker" -) - -const ( - ContextParamRegion = "region" - ContextParamProfile = "profile" -) - -func (b *Backend) CreateContextData(ctx context.Context, params map[string]string) (contextData interface{}, description string, err error) { - region, ok := params[ContextParamRegion] - if !ok { - return nil, "", fmt.Errorf("%q parameter is required", ContextParamRegion) - } - profile, ok := params[ContextParamProfile] - if !ok { - return nil, "", fmt.Errorf("%q parameter is required", ContextParamProfile) - } - err = b.api.CheckRequirements(ctx, region) - if err != nil { - return "", "", err - } - - return docker.AwsContext{ - Profile: profile, - Region: region, - }, "Amazon ECS context", nil -} diff --git a/ecs/pkg/amazon/backend/secrets.go b/ecs/pkg/amazon/backend/secrets.go deleted file mode 100644 index 6c86e95d..00000000 --- a/ecs/pkg/amazon/backend/secrets.go +++ /dev/null @@ -1,23 +0,0 @@ -package backend - -import ( - "context" - - "github.com/docker/ecs-plugin/pkg/compose" -) - -func (b Backend) CreateSecret(ctx context.Context, secret compose.Secret) (string, error) { - return b.api.CreateSecret(ctx, secret) -} - -func (b Backend) InspectSecret(ctx context.Context, id string) (compose.Secret, error) { - return b.api.InspectSecret(ctx, id) -} - -func (b Backend) ListSecrets(ctx context.Context) ([]compose.Secret, error) { - return b.api.ListSecrets(ctx) -} - -func (b Backend) DeleteSecret(ctx context.Context, id string, recover bool) error { - return b.api.DeleteSecret(ctx, id, recover) -} diff --git a/ecs/pkg/amazon/sdk/api.go b/ecs/pkg/amazon/sdk/api.go deleted file mode 100644 index af6dfed4..00000000 --- a/ecs/pkg/amazon/sdk/api.go +++ /dev/null @@ -1,42 +0,0 @@ -package sdk - -import ( - "context" - - cf "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/awslabs/goformation/v4/cloudformation" - "github.com/docker/ecs-plugin/pkg/compose" -) - -type API interface { - CheckRequirements(ctx context.Context, region string) error - - GetDefaultVPC(ctx context.Context) (string, error) - VpcExists(ctx context.Context, vpcID string) (bool, error) - GetSubNets(ctx context.Context, vpcID string) ([]string, error) - - StackExists(ctx context.Context, name string) (bool, error) - CreateStack(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) error - DeleteStack(ctx context.Context, name string) error - ListStackParameters(ctx context.Context, name string) (map[string]string, error) - ListStackResources(ctx context.Context, name string) ([]compose.StackResource, error) - GetStackID(ctx context.Context, name string) (string, error) - WaitStackComplete(ctx context.Context, name string, operation int) error - DescribeStackEvents(ctx context.Context, stackID string) ([]*cf.StackEvent, error) - CreateChangeSet(ctx context.Context, name string, template *cloudformation.Template, parameters map[string]string) (string, error) - UpdateStack(ctx context.Context, changeset string) error - - DescribeServices(ctx context.Context, cluster string, arns []string) ([]compose.ServiceStatus, error) - - LoadBalancerExists(ctx context.Context, arn string) (bool, error) - GetLoadBalancerURL(ctx context.Context, arn string) (string, error) - - ClusterExists(ctx context.Context, name string) (bool, error) - - GetLogs(ctx context.Context, name string, consumer compose.LogConsumer) error - - CreateSecret(ctx context.Context, secret compose.Secret) (string, error) - InspectSecret(ctx context.Context, id string) (compose.Secret, error) - ListSecrets(ctx context.Context) ([]compose.Secret, error) - DeleteSecret(ctx context.Context, id string, recover bool) error -} diff --git a/ecs/pkg/compose/api.go b/ecs/pkg/compose/api.go deleted file mode 100644 index 77f17055..00000000 --- a/ecs/pkg/compose/api.go +++ /dev/null @@ -1,26 +0,0 @@ -package compose - -import ( - "context" - "io" - - "github.com/awslabs/goformation/v4/cloudformation" - "github.com/compose-spec/compose-go/cli" - "github.com/compose-spec/compose-go/types" -) - -type API interface { - Up(ctx context.Context, options *cli.ProjectOptions) error - Down(ctx context.Context, options *cli.ProjectOptions) error - - CreateContextData(ctx context.Context, params map[string]string) (contextData interface{}, description string, err error) - - Convert(project *types.Project) (*cloudformation.Template, error) - Logs(ctx context.Context, options *cli.ProjectOptions, writer io.Writer) error - Ps(ctx context.Context, options *cli.ProjectOptions) ([]ServiceStatus, error) - - CreateSecret(ctx context.Context, secret Secret) (string, error) - InspectSecret(ctx context.Context, id string) (Secret, error) - ListSecrets(ctx context.Context) ([]Secret, error) - DeleteSecret(ctx context.Context, id string, recover bool) error -} diff --git a/ecs/pkg/compose/testdata/simple/compose-with-overrides.yaml b/ecs/pkg/compose/testdata/simple/compose-with-overrides.yaml deleted file mode 100644 index 3dc8a0b6..00000000 --- a/ecs/pkg/compose/testdata/simple/compose-with-overrides.yaml +++ /dev/null @@ -1,4 +0,0 @@ -version: "3" -services: - simple: - image: haproxy diff --git a/ecs/pkg/compose/testdata/simple/compose.yaml b/ecs/pkg/compose/testdata/simple/compose.yaml deleted file mode 100644 index 4b3f9af2..00000000 --- a/ecs/pkg/compose/testdata/simple/compose.yaml +++ /dev/null @@ -1,4 +0,0 @@ -version: "3" -services: - simple: - image: nginx \ No newline at end of file diff --git a/ecs/pkg/docker/contextStore.go b/ecs/pkg/docker/contextStore.go deleted file mode 100644 index 30a12de2..00000000 --- a/ecs/pkg/docker/contextStore.go +++ /dev/null @@ -1,76 +0,0 @@ -package docker - -import ( - "fmt" - - "github.com/docker/cli/cli/command" - cliconfig "github.com/docker/cli/cli/config" - "github.com/docker/cli/cli/context/store" - "github.com/mitchellh/mapstructure" -) - -const contextType = "aws" - -type TypeContext struct { - Type string -} - -func getter() interface{} { - return &TypeContext{} -} - -type AwsContext struct { - Profile string - Region string -} - -func NewContext(name string, awsContext interface{}) error { - _, err := NewContextWithStore(name, awsContext.(AwsContext), cliconfig.ContextStoreDir()) - return err -} - -func NewContextWithStore(name string, awsContext AwsContext, contextDirectory string) (store.Store, error) { - contextStore := initContextStore(contextDirectory) - endpoints := map[string]interface{}{ - "aws": awsContext, - "docker": awsContext, - } - - metadata := store.Metadata{ - Name: name, - Endpoints: endpoints, - Metadata: TypeContext{Type: contextType}, - } - return contextStore, contextStore.CreateOrUpdate(metadata) -} - -func initContextStore(contextDir string) store.Store { - config := store.NewConfig(getter) - return store.New(contextDir, config) -} - -func checkAwsContextExists(contextName string) (*AwsContext, error) { - if contextName == command.DefaultContextName { - return nil, fmt.Errorf("can't use \"%s\" with ECS command because it is not an AWS context", contextName) - } - contextStore := initContextStore(cliconfig.ContextStoreDir()) - metadata, err := contextStore.GetMetadata(contextName) - if err != nil { - return nil, err - } - endpoint := metadata.Endpoints["aws"] - awsContext := AwsContext{} - err = mapstructure.Decode(endpoint, &awsContext) - if err != nil { - return nil, err - } - if awsContext == (AwsContext{}) { - return nil, fmt.Errorf("can't use \"%s\" with ECS command because it is not an AWS context", contextName) - } - return &awsContext, nil -} - -func GetAwsContext(dockerCli command.Cli) (*AwsContext, error) { - contextName := dockerCli.CurrentContext() - return checkAwsContextExists(contextName) -} diff --git a/ecs/pkg/progress/plain.go b/ecs/pkg/progress/plain.go deleted file mode 100644 index 8e476807..00000000 --- a/ecs/pkg/progress/plain.go +++ /dev/null @@ -1,29 +0,0 @@ -package progress - -import ( - "context" - "fmt" - "io" -) - -type plainWriter struct { - out io.Writer - done chan bool -} - -func (p *plainWriter) Start(ctx context.Context) error { - select { - case <-ctx.Done(): - return ctx.Err() - case <-p.done: - return nil - } -} - -func (p *plainWriter) Event(e Event) { - fmt.Println(e.ID, e.Text, e.StatusText) -} - -func (p *plainWriter) Stop() { - p.done <- true -} diff --git a/ecs/pkg/progress/spinner.go b/ecs/pkg/progress/spinner.go deleted file mode 100644 index 695a5642..00000000 --- a/ecs/pkg/progress/spinner.go +++ /dev/null @@ -1,50 +0,0 @@ -package progress - -import ( - "runtime" - "time" -) - -type spinner struct { - time time.Time - index int - chars []string - stop bool - done string -} - -func newSpinner() *spinner { - chars := []string{ - "⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏", - } - done := "⠿" - - if runtime.GOOS == "windows" { - chars = []string{"-"} - done = "-" - } - - return &spinner{ - index: 0, - time: time.Now(), - chars: chars, - done: done, - } -} - -func (s *spinner) String() string { - if s.stop { - return s.done - } - - d := time.Since(s.time) - if d.Milliseconds() > 100 { - s.index = (s.index + 1) % len(s.chars) - } - - return s.chars[s.index] -} - -func (s *spinner) Stop() { - s.stop = true -} diff --git a/ecs/pkg/progress/tty.go b/ecs/pkg/progress/tty.go deleted file mode 100644 index 6877e096..00000000 --- a/ecs/pkg/progress/tty.go +++ /dev/null @@ -1,177 +0,0 @@ -package progress - -import ( - "context" - "fmt" - "io" - "runtime" - "strings" - "sync" - "time" - - "github.com/buger/goterm" - "github.com/morikuni/aec" -) - -type ttyWriter struct { - out io.Writer - events map[string]Event - eventIDs []string - repeated bool - numLines int - done chan bool - mtx *sync.RWMutex -} - -func (w *ttyWriter) Start(ctx context.Context) error { - ticker := time.NewTicker(100 * time.Millisecond) - - for { - select { - case <-ctx.Done(): - w.print() - return ctx.Err() - case <-w.done: - w.print() - return nil - case <-ticker.C: - w.print() - } - } -} - -func (w *ttyWriter) Stop() { - w.done <- true -} - -func (w *ttyWriter) Event(e Event) { - w.mtx.Lock() - defer w.mtx.Unlock() - if !StringContains(w.eventIDs, e.ID) { - w.eventIDs = append(w.eventIDs, e.ID) - } - if _, ok := w.events[e.ID]; ok { - last := w.events[e.ID] - switch e.Status { - case Done, Error: - if last.Status != e.Status { - last.stop() - } - } - last.Status = e.Status - last.Text = e.Text - last.StatusText = e.StatusText - w.events[e.ID] = last - } else { - e.startTime = time.Now() - e.spinner = newSpinner() - w.events[e.ID] = e - } -} - -func (w *ttyWriter) print() { - w.mtx.Lock() - defer w.mtx.Unlock() - if len(w.eventIDs) == 0 { - return - } - terminalWidth := goterm.Width() - b := aec.EmptyBuilder - for i := 0; i <= w.numLines; i++ { - b = b.Up(1) - } - if !w.repeated { - b = b.Down(1) - } - w.repeated = true - fmt.Fprint(w.out, b.Column(0).ANSI) - - // Hide the cursor while we are printing - fmt.Fprint(w.out, aec.Hide) - defer fmt.Fprint(w.out, aec.Show) - - firstLine := fmt.Sprintf("[+] Running %d/%d", numDone(w.events), w.numLines) - if w.numLines != 0 && numDone(w.events) == w.numLines { - firstLine = aec.Apply(firstLine, aec.BlueF) - } - fmt.Fprintln(w.out, firstLine) - - var statusPadding int - for _, v := range w.eventIDs { - l := len(fmt.Sprintf("%s %s", w.events[v].ID, w.events[v].Text)) - if statusPadding < l { - statusPadding = l - } - } - - numLines := 0 - for _, v := range w.eventIDs { - line := lineText(w.events[v], terminalWidth, statusPadding, runtime.GOOS != "windows") - // nolint: errcheck - fmt.Fprint(w.out, line) - numLines++ - } - - w.numLines = numLines -} - -func lineText(event Event, terminalWidth, statusPadding int, color bool) string { - endTime := time.Now() - if event.Status != Working { - endTime = event.endTime - } - - elapsed := endTime.Sub(event.startTime).Seconds() - - textLen := len(fmt.Sprintf("%s %s", event.ID, event.Text)) - padding := statusPadding - textLen - if padding < 0 { - padding = 0 - } - text := fmt.Sprintf(" %s %s %s%s %s", - event.spinner.String(), - event.ID, - event.Text, - strings.Repeat(" ", padding), - event.StatusText, - ) - timer := fmt.Sprintf("%.1fs\n", elapsed) - o := align(text, timer, terminalWidth) - - if color { - color := aec.WhiteF - if event.Status == Done { - color = aec.BlueF - } - if event.Status == Error { - color = aec.RedF - } - return aec.Apply(o, color) - } - - return o -} - -func numDone(events map[string]Event) int { - i := 0 - for _, e := range events { - if e.Status == Done { - i++ - } - } - return i -} - -func align(l, r string, w int) string { - return fmt.Sprintf("%-[2]*[1]s %[3]s", l, w-len(r)-1, r) -} - -// StringContains check if an array contains a specific value -func StringContains(array []string, needle string) bool { - for _, val := range array { - if val == needle { - return true - } - } - return false -} diff --git a/ecs/pkg/progress/writer.go b/ecs/pkg/progress/writer.go deleted file mode 100644 index a5b8e1cd..00000000 --- a/ecs/pkg/progress/writer.go +++ /dev/null @@ -1,112 +0,0 @@ -package progress - -import ( - "context" - "os" - "sync" - "time" - - "github.com/containerd/console" - "github.com/moby/term" - "golang.org/x/sync/errgroup" -) - -// EventStatus indicates the status of an action -type EventStatus int - -const ( - // Working means that the current task is working - Working EventStatus = iota - // Done means that the current task is done - Done - // Error means that the current task has errored - Error -) - -// Event reprensents a progress event -type Event struct { - ID string - Text string - Status EventStatus - StatusText string - Done bool - - startTime time.Time - endTime time.Time - spinner *spinner -} - -func (e *Event) stop() { - e.endTime = time.Now() - e.spinner.Stop() -} - -// Writer can write multiple progress events -type Writer interface { - Start(context.Context) error - Stop() - Event(Event) -} - -type writerKey struct{} - -// WithContextWriter adds the writer to the context -func WithContextWriter(ctx context.Context, writer Writer) context.Context { - return context.WithValue(ctx, writerKey{}, writer) -} - -// ContextWriter returns the writer from the context -func ContextWriter(ctx context.Context) Writer { - s, _ := ctx.Value(writerKey{}).(Writer) - return s -} - -type progressFunc func(context.Context) error - -// Run will run a writer and the progress function -// in parallel -func Run(ctx context.Context, pf progressFunc) error { - eg, _ := errgroup.WithContext(ctx) - w, err := NewWriter(os.Stderr) - if err != nil { - return err - } - eg.Go(func() error { - return w.Start(context.Background()) - }) - - ctx = WithContextWriter(ctx, w) - - eg.Go(func() error { - defer w.Stop() - return pf(ctx) - }) - - return eg.Wait() -} - -// NewWriter returns a new multi-progress writer -func NewWriter(out console.File) (Writer, error) { - _, isTerminal := term.GetFdInfo(out) - - if isTerminal { - con, err := console.ConsoleFromFile(out) - if err != nil { - return nil, err - } - - return &ttyWriter{ - out: con, - eventIDs: []string{}, - events: map[string]Event{}, - repeated: false, - done: make(chan bool), - mtx: &sync.RWMutex{}, - }, nil - } - - return &plainWriter{ - out: out, - done: make(chan bool), - }, nil -} diff --git a/ecs/pkg/amazon/sdk/sdk.go b/ecs/sdk.go similarity index 93% rename from ecs/pkg/amazon/sdk/sdk.go rename to ecs/sdk.go index 057e9fff..1bdcbecc 100644 --- a/ecs/pkg/amazon/sdk/sdk.go +++ b/ecs/sdk.go @@ -1,15 +1,13 @@ -package sdk +package ecs import ( "context" "fmt" + "github.com/docker/api/compose" "strings" "time" - cloudformation2 "github.com/docker/ecs-plugin/pkg/amazon/cloudformation" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" @@ -26,8 +24,6 @@ import ( "github.com/aws/aws-sdk-go/service/secretsmanager" "github.com/aws/aws-sdk-go/service/secretsmanager/secretsmanageriface" cf "github.com/awslabs/goformation/v4/cloudformation" - "github.com/docker/ecs-plugin/internal" - "github.com/docker/ecs-plugin/pkg/compose" "github.com/sirupsen/logrus" ) @@ -42,10 +38,7 @@ type sdk struct { SM secretsmanageriface.SecretsManagerAPI } -func NewAPI(sess *session.Session) API { - sess.Handlers.Build.PushBack(func(r *request.Request) { - request.AddToUserAgent(r, fmt.Sprintf("Docker CLI %s", internal.Version)) - }) +func NewSDK(sess *session.Session) sdk { return sdk{ ECS: ecs.New(sess), EC2: ec2.New(sess), @@ -166,7 +159,7 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) { func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template, parameters map[string]string) error { logrus.Debug("Create CloudFormation stack") - json, err := cloudformation2.Marshall(template) + json, err := Marshall(template) if err != nil { return err } @@ -194,7 +187,7 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template func (s sdk) CreateChangeSet(ctx context.Context, name string, template *cf.Template, parameters map[string]string) (string, error) { logrus.Debug("Create CloudFormation Changeset") - json, err := cloudformation2.Marshall(template) + json, err := Marshall(template) if err != nil { return "", err } @@ -251,9 +244,9 @@ func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) StackName: aws.String(name), } switch operation { - case compose.StackCreate: + case StackCreate: return s.CF.WaitUntilStackCreateCompleteWithContext(ctx, input) - case compose.StackDelete: + case StackDelete: return s.CF.WaitUntilStackDeleteCompleteWithContext(ctx, input) default: return fmt.Errorf("internal error: unexpected stack operation %d", operation) @@ -305,7 +298,7 @@ func (s sdk) ListStackParameters(ctx context.Context, name string) (map[string]s return parameters, nil } -func (s sdk) ListStackResources(ctx context.Context, name string) ([]compose.StackResource, error) { +func (s sdk) ListStackResources(ctx context.Context, name string) ([]StackResource, error) { // FIXME handle pagination res, err := s.CF.ListStackResourcesWithContext(ctx, &cloudformation.ListStackResourcesInput{ StackName: aws.String(name), @@ -314,9 +307,9 @@ func (s sdk) ListStackResources(ctx context.Context, name string) ([]compose.Sta return nil, err } - resources := []compose.StackResource{} + resources := []StackResource{} for _, r := range res.StackResourceSummaries { - resources = append(resources, compose.StackResource{ + resources = append(resources, StackResource{ LogicalID: aws.StringValue(r.LogicalResourceId), Type: aws.StringValue(r.ResourceType), ARN: aws.StringValue(r.PhysicalResourceId), @@ -334,7 +327,7 @@ func (s sdk) DeleteStack(ctx context.Context, name string) error { return err } -func (s sdk) CreateSecret(ctx context.Context, secret compose.Secret) (string, error) { +func (s sdk) CreateSecret(ctx context.Context, secret Secret) (string, error) { logrus.Debug("Create secret " + secret.Name) secretStr, err := secret.GetCredString() if err != nil { @@ -352,17 +345,17 @@ func (s sdk) CreateSecret(ctx context.Context, secret compose.Secret) (string, e return aws.StringValue(response.ARN), nil } -func (s sdk) InspectSecret(ctx context.Context, id string) (compose.Secret, error) { +func (s sdk) InspectSecret(ctx context.Context, id string) (Secret, error) { logrus.Debug("Inspect secret " + id) response, err := s.SM.DescribeSecret(&secretsmanager.DescribeSecretInput{SecretId: &id}) if err != nil { - return compose.Secret{}, err + return Secret{}, err } labels := map[string]string{} for _, tag := range response.Tags { labels[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value) } - secret := compose.Secret{ + secret := Secret{ ID: aws.StringValue(response.ARN), Name: aws.StringValue(response.Name), Labels: labels, @@ -373,13 +366,13 @@ func (s sdk) InspectSecret(ctx context.Context, id string) (compose.Secret, erro return secret, nil } -func (s sdk) ListSecrets(ctx context.Context) ([]compose.Secret, error) { +func (s sdk) ListSecrets(ctx context.Context) ([]Secret, error) { logrus.Debug("List secrets ...") response, err := s.SM.ListSecrets(&secretsmanager.ListSecretsInput{}) if err != nil { - return []compose.Secret{}, err + return []Secret{}, err } - var secrets []compose.Secret + var secrets []Secret for _, sec := range response.SecretList { @@ -391,7 +384,7 @@ func (s sdk) ListSecrets(ctx context.Context) ([]compose.Secret, error) { if sec.Description != nil { description = *sec.Description } - secrets = append(secrets, compose.Secret{ + secrets = append(secrets, Secret{ ID: *sec.ARN, Name: *sec.Name, Labels: labels, @@ -408,7 +401,7 @@ func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { return err } -func (s sdk) GetLogs(ctx context.Context, name string, consumer compose.LogConsumer) error { +func (s sdk) GetLogs(ctx context.Context, name string, consumer LogConsumer) error { logGroup := fmt.Sprintf("/docker-compose/%s", name) var startTime int64 for { diff --git a/ecs/secrets.go b/ecs/secrets.go new file mode 100644 index 00000000..b1a4578e --- /dev/null +++ b/ecs/secrets.go @@ -0,0 +1,21 @@ +package ecs + +import ( + "context" +) + +func (b ecsAPIService) CreateSecret(ctx context.Context, secret Secret) (string, error) { + return b.SDK.CreateSecret(ctx, secret) +} + +func (b ecsAPIService) InspectSecret(ctx context.Context, id string) (Secret, error) { + return b.SDK.InspectSecret(ctx, id) +} + +func (b ecsAPIService) ListSecrets(ctx context.Context) ([]Secret, error) { + return b.SDK.ListSecrets(ctx) +} + +func (b ecsAPIService) DeleteSecret(ctx context.Context, id string, recover bool) error { + return b.SDK.DeleteSecret(ctx, id, recover) +} diff --git a/ecs/secrets/Dockerfile b/ecs/secrets/Dockerfile index 638e113b..238f43dd 100644 --- a/ecs/secrets/Dockerfile +++ b/ecs/secrets/Dockerfile @@ -1,5 +1,5 @@ FROM golang:1.14.4-alpine AS builder -WORKDIR $GOPATH/src/github.com/docker/ecs-plugin/secrets +WORKDIR $GOPATH/src/github.com/docker/api/ecs/secrets COPY . . RUN GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /go/bin/secrets main/main.go diff --git a/ecs/secrets/main/main.go b/ecs/secrets/main/main.go index 703f88f8..c66c6de3 100644 --- a/ecs/secrets/main/main.go +++ b/ecs/secrets/main/main.go @@ -3,9 +3,8 @@ package main import ( "encoding/json" "fmt" + "github.com/docker/api/ecs/secrets" "os" - - "github.com/docker/ecs-plugin/secrets" ) const secretsFolder = "/run/secrets" diff --git a/ecs/pkg/amazon/backend/testdata/input/envfile b/ecs/testdata/input/envfile similarity index 100% rename from ecs/pkg/amazon/backend/testdata/input/envfile rename to ecs/testdata/input/envfile diff --git a/ecs/pkg/amazon/backend/testdata/input/simple-single-service.yaml b/ecs/testdata/input/simple-single-service.yaml similarity index 100% rename from ecs/pkg/amazon/backend/testdata/input/simple-single-service.yaml rename to ecs/testdata/input/simple-single-service.yaml diff --git a/ecs/pkg/amazon/backend/testdata/invalid_network_mode.yaml b/ecs/testdata/invalid_network_mode.yaml similarity index 100% rename from ecs/pkg/amazon/backend/testdata/invalid_network_mode.yaml rename to ecs/testdata/invalid_network_mode.yaml diff --git a/ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden b/ecs/testdata/simple/simple-cloudformation-conversion.golden similarity index 100% rename from ecs/pkg/amazon/backend/testdata/simple/simple-cloudformation-conversion.golden rename to ecs/testdata/simple/simple-cloudformation-conversion.golden diff --git a/ecs/tests/e2e_deploy_services_test.go b/ecs/tests/e2e_deploy_services_test.go deleted file mode 100644 index 32b6dab5..00000000 --- a/ecs/tests/e2e_deploy_services_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// +build e2e - -package tests - -import ( - "context" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/docker/ecs-plugin/pkg/amazon/sdk" - "github.com/docker/ecs-plugin/pkg/docker" - "gotest.tools/v3/assert" - "gotest.tools/v3/fs" - "gotest.tools/v3/golden" - "gotest.tools/v3/icmd" -) - -const ( - composeFileName = "compose.yaml" -) - -func TestE2eDeployServices(t *testing.T) { - cmd, cleanup, awsContext := dockerCli.createTestCmd() - defer cleanup() - - composeUpSimpleService(t, cmd, awsContext) -} - -func composeUpSimpleService(t *testing.T, cmd icmd.Cmd, awsContext docker.AwsContext) { - bgContext := context.Background() - composeYAML := golden.Get(t, "input/simple-single-service.yaml") - tmpDir := fs.NewDir(t, t.Name(), - fs.WithFile(composeFileName, "", fs.WithBytes(composeYAML)), - ) - // We can't use the file added in the tmp directory because it will drop if an assertion fails - defer composeDown(t, cmd, golden.Path("input/simple-single-service.yaml")) - defer tmpDir.Remove() - - cmd.Command = dockerCli.Command("ecs", "compose", "--file="+tmpDir.Join(composeFileName), "--project-name", t.Name(), "up") - icmd.RunCmd(cmd).Assert(t, icmd.Success) - - session, err := session.NewSessionWithOptions(session.Options{ - Profile: awsContext.Profile, - Config: aws.Config{ - Region: aws.String(awsContext.Region), - }, - }) - assert.NilError(t, err) - api := sdk.NewAPI(session) - arns, err := api.ListTasks(bgContext, t.Name(), t.Name()) - assert.NilError(t, err) - tasks, err := api.DescribeTasks(bgContext, t.Name(), arns...) - publicIps, err := api.GetPublicIPs(context.Background(), tasks[0].NetworkInterface) - assert.NilError(t, err) - for _, ip := range publicIps { - icmd.RunCommand("curl", "-I", "http://"+ip).Assert(t, icmd.Success) - } - -} - -func composeDown(t *testing.T, cmd icmd.Cmd, composeFile string) { - cmd.Command = dockerCli.Command("ecs", "compose", "--file="+composeFile, "--project-name", t.Name(), "down") - icmd.RunCmd(cmd).Assert(t, icmd.Success) -} diff --git a/ecs/tests/main_test.go b/ecs/tests/main_test.go deleted file mode 100644 index 5849e406..00000000 --- a/ecs/tests/main_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package tests - -import ( - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "runtime" - "testing" - - dockerConfigFile "github.com/docker/cli/cli/config/configfile" - "github.com/docker/ecs-plugin/pkg/docker" - "gotest.tools/v3/icmd" -) - -var ( - e2ePath = flag.String("e2e-path", ".", "Set path to the e2e directory") - dockerCliPath = os.Getenv("DOCKERCLI_BINARY") - dockerCli dockerCliCommand - testContextName = "testAwsContextToBeRemoved" -) - -type dockerCliCommand struct { - path string - cliPluginDir string -} - -type ConfigFileOperator func(configFile *dockerConfigFile.ConfigFile) - -func (d dockerCliCommand) createTestCmd(ops ...ConfigFileOperator) (icmd.Cmd, func(), docker.AwsContext) { - configDir, err := ioutil.TempDir("", "config") - if err != nil { - panic(err) - } - configFilePath := filepath.Join(configDir, "config.json") - config := dockerConfigFile.ConfigFile{ - CLIPluginsExtraDirs: []string{ - d.cliPluginDir, - }, - Filename: configFilePath, - } - for _, op := range ops { - op(&config) - } - configFile, err := os.Create(configFilePath) - if err != nil { - panic(err) - } - defer configFile.Close() - err = json.NewEncoder(configFile).Encode(config) - if err != nil { - panic(err) - } - - awsContext := docker.AwsContext{ - Profile: "sandbox.devtools.developer", - Region: "eu-west-3", - } - testStore, err := docker.NewContextWithStore(testContextName, awsContext, filepath.Join(configDir, "contexts")) - if err != nil { - panic(err) - } - cleanup := func() { - fmt.Println("cleanup") - testStore.Remove(testContextName) - os.RemoveAll(configDir) - } - env := append(os.Environ(), - "DOCKER_CONFIG="+configDir, - "DOCKER_CLI_EXPERIMENTAL=enabled") // TODO: Remove this once docker ecs plugin is no more experimental - return icmd.Cmd{Env: env}, cleanup, awsContext -} - -func (d dockerCliCommand) Command(args ...string) []string { - return append([]string{d.path, "--context", testContextName}, args...) -} - -func TestMain(m *testing.M) { - flag.Parse() - if err := os.Chdir(*e2ePath); err != nil { - panic(err) - } - cwd, err := os.Getwd() - if err != nil { - panic(err) - } - dockerEcs := os.Getenv("DOCKERECS_BINARY") - if dockerEcs == "" { - dockerEcs = filepath.Join(cwd, "../dist/docker-ecs") - } - dockerEcs, err = filepath.Abs(dockerEcs) - if err != nil { - panic(err) - } - if dockerCliPath == "" { - dockerCliPath = "docker" - } else { - dockerCliPath, err = filepath.Abs(dockerCliPath) - if err != nil { - panic(err) - } - } - // Prepare docker cli to call the docker-ecs plugin binary: - // - Create a symbolic link with the dockerEcs binary to the plugin directory - cliPluginDir, err := ioutil.TempDir("", "configContent") - if err != nil { - panic(err) - } - defer os.RemoveAll(cliPluginDir) - createDockerECSSymLink(dockerEcs, cliPluginDir) - - dockerCli = dockerCliCommand{path: dockerCliPath, cliPluginDir: cliPluginDir} - os.Exit(m.Run()) -} - -func createDockerECSSymLink(dockerEcs, configDir string) { - dockerEcsExecName := "docker-ecs" - if runtime.GOOS == "windows" { - dockerEcsExecName += ".exe" - } - if err := os.Symlink(dockerEcs, filepath.Join(configDir, dockerEcsExecName)); err != nil { - panic(err) - } -} diff --git a/ecs/tests/testdata/input/simple-single-service.yaml b/ecs/tests/testdata/input/simple-single-service.yaml deleted file mode 100644 index 95f30b1d..00000000 --- a/ecs/tests/testdata/input/simple-single-service.yaml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3" -services: - simple: - image: nginx - ports: - - 80:80 \ No newline at end of file diff --git a/ecs/tests/testdata/plugin-usage.golden b/ecs/tests/testdata/plugin-usage.golden deleted file mode 100644 index 8114b6f4..00000000 --- a/ecs/tests/testdata/plugin-usage.golden +++ /dev/null @@ -1,14 +0,0 @@ - -Usage: docker ecs COMMAND - -run multi-container applications on Amazon ECS. - -Management Commands: - compose - secret Manages secrets - -Commands: - setup - version Show version. - -Run 'docker ecs COMMAND --help' for more information on a command. diff --git a/ecs/tests/version_test.go b/ecs/tests/version_test.go deleted file mode 100644 index ad4d6455..00000000 --- a/ecs/tests/version_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package tests - -import ( - "strings" - "testing" - - "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" -) - -func TestVersionIsSet(t *testing.T) { - cmd, cleanup, _ := dockerCli.createTestCmd() - defer cleanup() - - cmd.Command = dockerCli.Command("ecs", "version") - out := icmd.RunCmd(cmd).Assert(t, icmd.Success).Stdout() - assert.Check(t, !strings.Contains(out, "unknown")) -} diff --git a/ecs/pkg/compose/types.go b/ecs/types.go similarity index 78% rename from ecs/pkg/compose/types.go rename to ecs/types.go index 807b9c7a..f1e30073 100644 --- a/ecs/pkg/compose/types.go +++ b/ecs/types.go @@ -1,4 +1,4 @@ -package compose +package ecs import "encoding/json" @@ -9,22 +9,6 @@ type StackResource struct { Status string } -type LoadBalancer struct { - URL string - TargetPort int - PublishedPort int - Protocol string -} - -type ServiceStatus struct { - ID string - Name string - Replicas int - Desired int - Ports []string - LoadBalancers []LoadBalancer -} - const ( StackCreate = iota StackUpdate diff --git a/ecs/pkg/amazon/backend/up.go b/ecs/up.go similarity index 65% rename from ecs/pkg/amazon/backend/up.go rename to ecs/up.go index 098caaac..dc36ecde 100644 --- a/ecs/pkg/amazon/backend/up.go +++ b/ecs/up.go @@ -1,4 +1,4 @@ -package backend +package ecs import ( "context" @@ -9,16 +9,15 @@ import ( "github.com/compose-spec/compose-go/cli" "github.com/compose-spec/compose-go/types" - "github.com/docker/ecs-plugin/pkg/compose" ) -func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { +func (b *ecsAPIService) Up(ctx context.Context, options *cli.ProjectOptions) error { project, err := cli.ProjectFromOptions(options) if err != nil { return err } - err = b.api.CheckRequirements(ctx, b.Region) + err = b.SDK.CheckRequirements(ctx, b.Region) if err != nil { return err } @@ -38,7 +37,7 @@ func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { return err } - subNets, err := b.api.GetSubNets(ctx, vpc) + subNets, err := b.SDK.GetSubNets(ctx, vpc) if err != nil { return err } @@ -59,23 +58,23 @@ func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { ParameterLoadBalancerARN: lb, } - update, err := b.api.StackExists(ctx, project.Name) + update, err := b.SDK.StackExists(ctx, project.Name) if err != nil { return err } - operation := compose.StackCreate + operation := StackCreate if update { - operation = compose.StackUpdate - changeset, err := b.api.CreateChangeSet(ctx, project.Name, template, parameters) + operation = StackUpdate + changeset, err := b.SDK.CreateChangeSet(ctx, project.Name, template, parameters) if err != nil { return err } - err = b.api.UpdateStack(ctx, changeset) + err = b.SDK.UpdateStack(ctx, changeset) if err != nil { return err } } else { - err = b.api.CreateStack(ctx, project.Name, template, parameters) + err = b.SDK.CreateStack(ctx, project.Name, template, parameters) if err != nil { return err } @@ -93,11 +92,11 @@ func (b *Backend) Up(ctx context.Context, options *cli.ProjectOptions) error { return err } -func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, error) { +func (b ecsAPIService) GetVPC(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if vpc, ok := project.Extensions[compose.ExtensionVPC]; ok { + if vpc, ok := project.Extensions[ExtensionVPC]; ok { vpcID := vpc.(string) - ok, err := b.api.VpcExists(ctx, vpcID) + ok, err := b.SDK.VpcExists(ctx, vpcID) if err != nil { return "", err } @@ -106,18 +105,18 @@ func (b Backend) GetVPC(ctx context.Context, project *types.Project) (string, er } return vpcID, nil } - defaultVPC, err := b.api.GetDefaultVPC(ctx) + defaultVPC, err := b.SDK.GetDefaultVPC(ctx) if err != nil { return "", err } return defaultVPC, nil } -func (b Backend) GetLoadBalancer(ctx context.Context, project *types.Project) (string, error) { +func (b ecsAPIService) GetLoadBalancer(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if ext, ok := project.Extensions[compose.ExtensionLB]; ok { + if ext, ok := project.Extensions[ExtensionLB]; ok { lb := ext.(string) - ok, err := b.api.LoadBalancerExists(ctx, lb) + ok, err := b.SDK.LoadBalancerExists(ctx, lb) if err != nil { return "", err } @@ -129,11 +128,11 @@ func (b Backend) GetLoadBalancer(ctx context.Context, project *types.Project) (s return "", nil } -func (b Backend) GetCluster(ctx context.Context, project *types.Project) (string, error) { +func (b ecsAPIService) GetCluster(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if ext, ok := project.Extensions[compose.ExtensionCluster]; ok { + if ext, ok := project.Extensions[ExtensionCluster]; ok { cluster := ext.(string) - ok, err := b.api.ClusterExists(ctx, cluster) + ok, err := b.SDK.ClusterExists(ctx, cluster) if err != nil { return "", err } diff --git a/ecs/pkg/amazon/backend/wait.go b/ecs/wait.go similarity index 75% rename from ecs/pkg/amazon/backend/wait.go rename to ecs/wait.go index dcc26772..4ca41d85 100644 --- a/ecs/pkg/amazon/backend/wait.go +++ b/ecs/wait.go @@ -1,23 +1,22 @@ -package backend +package ecs import ( "context" "fmt" + "github.com/docker/api/progress" "sort" "strings" "time" "github.com/aws/aws-sdk-go/aws" - "github.com/docker/ecs-plugin/pkg/compose" - "github.com/docker/ecs-plugin/pkg/progress" ) -func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operation int) error { +func (b *ecsAPIService) WaitStackCompletion(ctx context.Context, name string, operation int) error { knownEvents := map[string]struct{}{} // progress writer w := progress.ContextWriter(ctx) // Get the unique Stack ID so we can collect events without getting some from previous deployments with same name - stackID, err := b.api.GetStackID(ctx, name) + stackID, err := b.SDK.GetStackID(ctx, name) if err != nil { return err } @@ -25,7 +24,7 @@ func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operatio ticker := time.NewTicker(1 * time.Second) done := make(chan bool) go func() { - b.api.WaitStackComplete(ctx, stackID, operation) //nolint:errcheck + b.SDK.WaitStackComplete(ctx, stackID, operation) //nolint:errcheck ticker.Stop() done <- true }() @@ -38,7 +37,7 @@ func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operatio completed = true case <-ticker.C: } - events, err := b.api.DescribeStackEvents(ctx, stackID) + events, err := b.SDK.DescribeStackEvents(ctx, stackID) if err != nil { return err } @@ -60,23 +59,23 @@ func (b *Backend) WaitStackCompletion(ctx context.Context, name string, operatio switch status { case "CREATE_COMPLETE": - if operation == compose.StackCreate { + if operation == StackCreate { progressStatus = progress.Done } case "UPDATE_COMPLETE": - if operation == compose.StackUpdate { + if operation == StackUpdate { progressStatus = progress.Done } case "DELETE_COMPLETE": - if operation == compose.StackDelete { + if operation == StackDelete { progressStatus = progress.Done } default: if strings.HasSuffix(status, "_FAILED") { progressStatus = progress.Error if stackErr == nil { - operation = compose.StackDelete + operation = StackDelete stackErr = fmt.Errorf(reason) } } diff --git a/ecs/pkg/compose/x.go b/ecs/x.go similarity index 97% rename from ecs/pkg/compose/x.go rename to ecs/x.go index 3a4ffdb6..9dee6a36 100644 --- a/ecs/pkg/compose/x.go +++ b/ecs/x.go @@ -1,4 +1,4 @@ -package compose +package ecs const ( ExtensionSecurityGroup = "x-aws-securitygroup" diff --git a/example/backend.go b/example/backend.go index 9cb54004..c9a10c62 100644 --- a/example/backend.go +++ b/example/backend.go @@ -22,9 +22,9 @@ import ( "context" "errors" "fmt" + "io" "github.com/compose-spec/compose-go/cli" - ecstypes "github.com/docker/ecs-plugin/pkg/compose" "github.com/docker/api/backend" "github.com/docker/api/compose" @@ -117,8 +117,8 @@ func (cs *containerService) Delete(ctx context.Context, id string, request conta type composeService struct{} -func (cs *composeService) Up(ctx context.Context, opts cli.ProjectOptions) error { - prj, err := cli.ProjectFromOptions(&opts) +func (cs *composeService) Up(ctx context.Context, opts *cli.ProjectOptions) error { + prj, err := cli.ProjectFromOptions(opts) if err != nil { return err } @@ -126,8 +126,8 @@ func (cs *composeService) Up(ctx context.Context, opts cli.ProjectOptions) error return nil } -func (cs *composeService) Down(ctx context.Context, opts cli.ProjectOptions) error { - prj, err := cli.ProjectFromOptions(&opts) +func (cs *composeService) Down(ctx context.Context, opts *cli.ProjectOptions) error { + prj, err := cli.ProjectFromOptions(opts) if err != nil { return err } @@ -135,10 +135,10 @@ func (cs *composeService) Down(ctx context.Context, opts cli.ProjectOptions) err return nil } -func (cs *composeService) Ps(ctx context.Context, opts cli.ProjectOptions) ([]ecstypes.ServiceStatus, error) { +func (cs *composeService) Ps(ctx context.Context, opts *cli.ProjectOptions) ([]compose.ServiceStatus, error) { return nil, errdefs.ErrNotImplemented } -func (cs *composeService) Logs(ctx context.Context, opts cli.ProjectOptions) error { +func (cs *composeService) Logs(ctx context.Context, opts *cli.ProjectOptions, w io.Writer) error { return errdefs.ErrNotImplemented } diff --git a/go.mod b/go.mod index f58ea079..5dd55cc5 100644 --- a/go.mod +++ b/go.mod @@ -20,25 +20,26 @@ require ( github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 github.com/Microsoft/hcsshim v0.8.9 // indirect github.com/aws/aws-sdk-go v1.34.2 + github.com/awslabs/goformation/v4 v4.14.0 github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 github.com/containerd/console v1.0.0 github.com/containerd/containerd v1.3.5 // indirect github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8 + github.com/docker/distribution v0.0.0-00010101000000-000000000000 // indirect github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible - github.com/docker/ecs-plugin v1.0.0-beta.4 + github.com/docker/docker-credential-helpers v0.6.3 // indirect github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee // indirect github.com/gobwas/pool v0.2.0 // indirect github.com/gobwas/ws v1.0.3 - github.com/gogo/googleapis v1.4.0 // indirect github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.1 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 // indirect github.com/hashicorp/go-multierror v1.1.0 - github.com/hashicorp/go-version v1.2.1 // indirect + github.com/joho/godotenv v1.3.0 github.com/moby/term v0.0.0-20200611042045-63b9a826fb74 github.com/morikuni/aec v1.0.0 github.com/onsi/gomega v1.10.1 // indirect @@ -46,6 +47,7 @@ require ( github.com/opencontainers/runc v0.1.1 // indirect github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.6.0 + github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 diff --git a/go.sum b/go.sum index 37a78fae..56d3d87b 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c/go.mod h1:hSVuE3qU7grINVSwrmzHfpg9k87ALBk+XaualNyUzI4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/AlecAivazis/survey/v2 v2.1.0 h1:AT4+23hOFopXYZaNGugbk7MWItkz0SfTmH/Hk92KeeE= -github.com/AlecAivazis/survey/v2 v2.1.0/go.mod h1:9FJRdMdDm8rnT+zHVbvQT2RTSTLq0Ttd6q3Vl2fahjk= github.com/AlecAivazis/survey/v2 v2.1.1 h1:LEMbHE0pLj75faaVEKClEX1TM4AJmmnOh9eimREzLWI= github.com/AlecAivazis/survey/v2 v2.1.1/go.mod h1:9FJRdMdDm8rnT+zHVbvQT2RTSTLq0Ttd6q3Vl2fahjk= github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= @@ -26,12 +22,8 @@ github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKn github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.9.0 h1:SigMbuFNuKgc1xcGhaeapbh+8fgsu+GxgDRFyg7f5lM= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.1 h1:xjPqigMQe2+0DAJ5A6MLUPp5D2r2Io8qHCuCMMI/yJU= -github.com/Azure/go-autorest/autorest/adal v0.9.1/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.2 h1:Aze/GQeAN1RRbGmnUJvUj+tFGBzFdIg3293/A9rbxC4= github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.0 h1:nSMjYIe24eBYasAIxt859TxyXef/IqoH+8/g4+LmcVs= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.0/go.mod h1:QRTvSZQpxqm8mSErhnbI+tANIBAKP7B+UIE2z4ypUO0= github.com/Azure/go-autorest/autorest/azure/auth v0.5.1 h1:bvUhZciHydpBxBmCheUgxxbSwJy7xcfjkUsjUcqSojc= github.com/Azure/go-autorest/autorest/azure/auth v0.5.1/go.mod h1:ea90/jvmnAwDrSooLH4sRIehEPtG/EPUXavDh31MnA4= github.com/Azure/go-autorest/autorest/azure/cli v0.4.0 h1:Ml+UCrnlKD+cJmSzrZ/RDcDw86NjkRUpnFh7V5JUhzU= @@ -43,6 +35,7 @@ github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxB github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.4.0 h1:z20OWOSG5aCye0HEkDp6TPmP17ZcfeMxPi6HnSALa8c= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= @@ -58,11 +51,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= -github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/Microsoft/hcsshim v0.8.9 h1:VrfodqvztU8YSOvygU+DN1BGaSGxmrNfqOv5oOuX2Bk= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= @@ -70,59 +60,32 @@ github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= -github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= -github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.30.22/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.33.21 h1:ziUemjajvLABlnJFe+8sM3fpqlg/DNA4944rUZ05PhY= -github.com/aws/aws-sdk-go v1.33.21/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.2 h1:9vCknCdTAmmV4ht7lPuda7aJXzllXwEQyCMZKJHjBrM= github.com/aws/aws-sdk-go v1.34.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/awslabs/goformation/v4 v4.8.0 h1:UiUhyokRy3suEqBXTnipvY8klqY3Eyl4GCH17brraEc= -github.com/awslabs/goformation/v4 v4.8.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= +github.com/awslabs/goformation/v4 v4.14.0 h1:E2Pet9eIqA4qzt3dzzzE4YN83V4Kyfbcio0VokBC9TA= +github.com/awslabs/goformation/v4 v4.14.0/go.mod h1:GcJULxCJfloT+3pbqCluXftdEK2AD/UqpS3hkaaBntg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= -github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 h1:gfAMKE626QEuKG3si0pdTRcr/YEbBoxY+3GOH3gWvl4= github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129/go.mod h1:u9UyCz2eTrSGy6fbupqJ54eY5c4IC8gREQ1053dK12U= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/bugsnag-go v1.5.3 h1:yeRUT3mUE13jL1tGwvoQsKdVbAsQx9AJ+fqahKveP04= -github.com/bugsnag/bugsnag-go v1.5.3/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA= -github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= -github.com/cloudflare/cfssl v1.4.1 h1:vScfU2DrIUI9VPHBVeeAQ0q5A+9yshO1Gz+3QoUQiKw= -github.com/cloudflare/cfssl v1.4.1/go.mod h1:KManx/OJPb5QY+y0+o/898AMcM128sF0bURvoVUSjTo= -github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= -github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 h1:F+YIkKDMHdgZBacawhFY1P9RAIgO+6uv2te6hjsjzF0= github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= @@ -131,14 +94,11 @@ github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqh github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v1.0.0 h1:fU3UuQapBs+zLJu82NhR11Rif1ny2zfMMAyPJzSN5tQ= github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.5 h1:l0iDHQtFwcOUmOvdepI6BB67q7beT6sRp2JYsfHS08c= github.com/containerd/containerd v1.3.5/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb h1:nXPkFq8X1a9ycY3GYQpFNxHh3j2JgY7zDZfq2EXMIzk= -github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 h1:PUD50EuOMkXVcpBIA/R95d56duJR9VxhwncsFbNnxW4= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= @@ -148,21 +108,16 @@ github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd h1:JNn81o/xG+8N github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= -github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= @@ -171,20 +126,14 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8 h1:JRquW4uqIU+eSilDhuo9X9QFX4NEmGj5B1x97ZA8djM= github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20200708230824-53e18a9d9bfe h1:pni13lAFm1g4cjHU6c3n4qGvvJGZQK4VvKRKMseQ42E= github.com/docker/distribution v0.0.0-20200708230824-53e18a9d9bfe/go.mod h1:Oqz4IonmMNc2N7GqfTL2xkhCQx0yS6nR+HrOZJnmKIk= -github.com/docker/docker v1.4.2-0.20200128034134-2ebaeef943cc/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible h1:G2hY8RD7jB9QaSmcb8mYEIg8QbEvVAB7se8+lXHZHfg= github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/ecs-plugin v1.0.0-beta.4 h1:hZKojW0tqsdhJjfMKPw6piMw/GJgfX6CVXd1YUuXLg4= -github.com/docker/ecs-plugin v1.0.0-beta.4/go.mod h1:1YaNZwrNr0dFjTP3v7zwepluaZgVNV94s0M6fL+i/iA= -github.com/docker/go v1.5.1-1 h1:hr4w35acWBPhGBXlzPoHpmZ/ygPjnmFVxGxxGnMyP7k= -github.com/docker/go v1.5.1-1/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= @@ -194,26 +143,18 @@ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHz github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -225,16 +166,10 @@ github.com/gobwas/ws v1.0.3 h1:ZOigqf7iBxkA4jdQ3am7ATzdlOFp9YzA6NmuvEEZc9g= github.com/gobwas/ws v1.0.3/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= -github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v1.4.0 h1:zgVt4UpGxcqVOw97aRGxT4svlcmdK35fynLNctY32zI= -github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -251,8 +186,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE= -github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -269,7 +202,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -277,16 +209,10 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmg github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= -github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -300,19 +226,10 @@ github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q= -github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= -github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -320,18 +237,12 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= -github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/kisom/goutils v1.1.0/go.mod h1:+UBTfd78habUYWFbNWTJNG+jNG/i/lGURakr4A/yNRw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -344,38 +255,21 @@ github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c= -github.com/lib/pq v0.0.0-20180201184707-88edab080323/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= -github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/manifoldco/promptui v0.7.0 h1:3l11YT8tm9MnwGFQ4kETwkzpAwY2Jt9lCrumCUW4+z4= -github.com/manifoldco/promptui v0.7.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= -github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -390,26 +284,19 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/gomega v1.2.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -421,11 +308,9 @@ github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJ github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700 h1:eNUVfm/RFLIi1G7flU5/ZRTHvd4kcVuzfRnL6OFlzCI= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -450,11 +335,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b h1:jUK33OXuZP/l6babJtnLo1qsGvq6G9so9KMflGAm4YA= github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b/go.mod h1:8458kAagoME2+LN5//WxE71ysZ3B7r22fdgb7qVmXSY= @@ -462,9 +344,7 @@ github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522 h1:fOCp11H0yuyAt2wql github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522/go.mod h1:tQTYKOQgxoH3v6dEmdHiz4JG+nbxWwM5fgPQUpSZqVQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= @@ -480,18 +360,14 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -504,43 +380,23 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/theupdateframework/notary v0.6.1 h1:7wshjstgS9x9F5LuB1L5mBI2xNMObWqjz+cjWoom6l0= -github.com/theupdateframework/notary v0.6.1/go.mod h1:MOfgIfmox8s7/7fduvB2xyPPMJCrjRLRizA8OFwpnKY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg2K1HrLU= -github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= -github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20190729165852-9051775e6a2e h1:mvOa4+/DXStR4ZXOks/UsjeFdn5O5JpLUtzqk9U8xXw= -github.com/zmap/zcrypto v0.0.0-20190729165852-9051775e6a2e/go.mod h1:w7kd3qXHh8FNaczNjslXqvFQiv5mMWRXlL9klTUAHc8= -github.com/zmap/zlint v0.0.0-20190806154020-fd021b4cfbeb h1:vxqkjztXSaPVDc8FQCdHTaejm2x747f6yPbnu1h2xkg= -github.com/zmap/zlint v0.0.0-20190806154020-fd021b4cfbeb/go.mod h1:29UiAJNsiVdvTBFCJW8e3q6dcDbOoPkhMgttOSCIMMY= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= @@ -548,14 +404,9 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -563,7 +414,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -598,8 +448,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -611,7 +459,6 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= @@ -621,8 +468,6 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -665,25 +510,14 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/dancannon/gorethink.v3 v3.0.5 h1:/g7PWP7zUS6vSNmHSDbjCHQh1Rqn8Jy6zSMQxAsBSMQ= -gopkg.in/dancannon/gorethink.v3 v3.0.5/go.mod h1:GXsi1e3N2OcKhcP6nsYABTiUejbWMFO4GY5a4pEaeEc= -gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= -gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y2O4MU= -gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= -gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.58.0 h1:VdDvTzv/005R8vEFyQ56bpEnOKTNPbpJhL0VCohxlQw= gopkg.in/ini.v1 v1.58.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -705,6 +539,3 @@ gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= -vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= From 91427d0492a94af23f9733f014fefcc6d9296d92 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 18 Aug 2020 08:58:37 +0200 Subject: [PATCH 200/205] introduce `Convert` in compose API Signed-off-by: Nicolas De Loof --- aci/backend.go | 4 ++++ compose/api.go | 2 ++ ecs/cloudformation.go | 18 ++++++++++++++++-- ecs/convert.go | 2 +- ecs/up.go | 2 +- example/backend.go | 4 ++++ 6 files changed, 28 insertions(+), 4 deletions(-) diff --git a/aci/backend.go b/aci/backend.go index fc5fe758..77879dfe 100644 --- a/aci/backend.go +++ b/aci/backend.go @@ -451,6 +451,10 @@ func (cs *aciComposeService) Logs(ctx context.Context, opts *cli.ProjectOptions, return errdefs.ErrNotImplemented } +func Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) { + return nil, errdefs.ErrNotImplemented +} + type aciCloudService struct { loginService login.AzureLoginServiceAPI } diff --git a/compose/api.go b/compose/api.go index d80050f0..ec055e7e 100644 --- a/compose/api.go +++ b/compose/api.go @@ -33,6 +33,8 @@ type Service interface { Logs(ctx context.Context, opts *cli.ProjectOptions, w io.Writer) error // Ps executes the equivalent to a `compose ps` Ps(ctx context.Context, opts *cli.ProjectOptions) ([]ServiceStatus, error) + // Convert translate compose model into backend's native format + Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) } diff --git a/ecs/cloudformation.go b/ecs/cloudformation.go index dda69f00..85f40a6a 100644 --- a/ecs/cloudformation.go +++ b/ecs/cloudformation.go @@ -1,7 +1,9 @@ package ecs import ( + "context" "fmt" + "github.com/compose-spec/compose-go/cli" "github.com/docker/api/compose" "io/ioutil" "regexp" @@ -33,8 +35,20 @@ const ( ParameterLoadBalancerARN = "ParameterLoadBalancerARN" ) +func (b *ecsAPIService) Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) { + project, err := cli.ProjectFromOptions(opts) + if err != nil { + return nil, err + } + template, err := b.convert(project) + if err != nil { + return nil, err + } + return Marshall(template) +} + // Convert a compose project into a CloudFormation template -func (b ecsAPIService) Convert(project *types.Project) (*cloudformation.Template, error) { +func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Template, error) { var checker compatibility.Checker = &FargateCompatibilityChecker{ compatibility.AllowList{ Supported: compatibleComposeAttributes, @@ -128,7 +142,7 @@ func (b ecsAPIService) Convert(project *types.Project) (*cloudformation.Template for _, service := range project.Services { - definition, err := Convert(project, service) + definition, err := convert(project, service) if err != nil { return nil, err } diff --git a/ecs/convert.go b/ecs/convert.go index 62eca65a..17db6064 100644 --- a/ecs/convert.go +++ b/ecs/convert.go @@ -23,7 +23,7 @@ import ( const secretsInitContainerImage = "docker/ecs-secrets-sidecar" -func Convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { +func convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefinition, error) { cpu, mem, err := toLimits(service) if err != nil { return nil, err diff --git a/ecs/up.go b/ecs/up.go index dc36ecde..44d16227 100644 --- a/ecs/up.go +++ b/ecs/up.go @@ -27,7 +27,7 @@ func (b *ecsAPIService) Up(ctx context.Context, options *cli.ProjectOptions) err return err } - template, err := b.Convert(project) + template, err := b.convert(project) if err != nil { return err } diff --git a/example/backend.go b/example/backend.go index c9a10c62..3206926e 100644 --- a/example/backend.go +++ b/example/backend.go @@ -142,3 +142,7 @@ func (cs *composeService) Ps(ctx context.Context, opts *cli.ProjectOptions) ([]c func (cs *composeService) Logs(ctx context.Context, opts *cli.ProjectOptions, w io.Writer) error { return errdefs.ErrNotImplemented } + +func Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) { + return nil, errdefs.ErrNotImplemented +} From 6e6a11aa738a5bac3be1617ead5f57eed394f27b Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 18 Aug 2020 09:13:58 +0200 Subject: [PATCH 201/205] Introduce Secrets API Signed-off-by: Nicolas De Loof --- aci/backend.go | 8 ++++++- backend/backend.go | 3 ++- client/client.go | 6 ++++++ ecs/backend.go | 5 +++++ ecs/cloudformation_test.go | 16 +++++++------- ecs/logs.go | 5 +++-- ecs/sdk.go | 38 +++++++++++++++++++++++----------- ecs/secrets.go | 10 +++++---- example/backend.go | 7 ++++++- go.mod | 6 +++--- go.sum | 26 +++++++++++++++++------ local/backend.go | 5 +++++ ecs/types.go => secrets/api.go | 26 +++++++++-------------- 13 files changed, 108 insertions(+), 53 deletions(-) rename ecs/types.go => secrets/api.go (68%) diff --git a/aci/backend.go b/aci/backend.go index 77879dfe..275d7d91 100644 --- a/aci/backend.go +++ b/aci/backend.go @@ -19,6 +19,7 @@ package aci import ( "context" "fmt" + "github.com/docker/api/secrets" "io" "net/http" "strconv" @@ -127,6 +128,11 @@ func (a *aciAPIService) ComposeService() compose.Service { return a.aciComposeService } +func (a *aciAPIService) SecretsService() secrets.Service { + return nil +} + + type aciContainerService struct { ctx store.AciContext } @@ -451,7 +457,7 @@ func (cs *aciComposeService) Logs(ctx context.Context, opts *cli.ProjectOptions, return errdefs.ErrNotImplemented } -func Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) { +func (cs *aciComposeService) Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) { return nil, errdefs.ErrNotImplemented } diff --git a/backend/backend.go b/backend/backend.go index 53b8df17..b5d5f9d5 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -20,7 +20,7 @@ import ( "context" "errors" "fmt" - + "github.com/docker/api/secrets" "github.com/sirupsen/logrus" "github.com/docker/api/compose" @@ -51,6 +51,7 @@ var backends = struct { // Service aggregates the service interfaces type Service interface { ContainerService() containers.Service + SecretsService() secrets.Service ComposeService() compose.Service } diff --git a/client/client.go b/client/client.go index f099c84a..86230505 100644 --- a/client/client.go +++ b/client/client.go @@ -18,6 +18,7 @@ package client import ( "context" + "github.com/docker/api/secrets" "github.com/docker/api/context/cloud" @@ -69,3 +70,8 @@ func (c *Client) ContainerService() containers.Service { func (c *Client) ComposeService() compose.Service { return c.bs.ComposeService() } + +// ComposeService returns the backend service for the current context +func (c *Client) SecretsService() secrets.Service { + return c.bs.SecretsService() +} diff --git a/ecs/backend.go b/ecs/backend.go index fd0d4685..a152f1a0 100644 --- a/ecs/backend.go +++ b/ecs/backend.go @@ -17,6 +17,7 @@ import ( "context" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" + "github.com/docker/api/secrets" "github.com/docker/api/backend" "github.com/docker/api/compose" @@ -88,6 +89,10 @@ func (a *ecsAPIService) ComposeService() compose.Service { return a } +func (a *ecsAPIService) SecretsService() secrets.Service { + return a +} + func getCloudService() (cloud.Service, error) { return ecsCloudService{}, nil } diff --git a/ecs/cloudformation_test.go b/ecs/cloudformation_test.go index e93dc86b..3943ee1f 100644 --- a/ecs/cloudformation_test.go +++ b/ecs/cloudformation_test.go @@ -2,6 +2,7 @@ package ecs import ( "fmt" + "github.com/docker/api/compose" "reflect" "testing" @@ -249,7 +250,8 @@ services: cpus: '0.5' memory: 2043248M `) - _, err := Backend{}.Convert(model) + backend := &ecsAPIService{} + _, err := backend.convert(model) assert.ErrorContains(t, err, "the resources requested are not supported by ECS/Fargate") } @@ -331,11 +333,10 @@ services: } func convertResultAsString(t *testing.T, project *types.Project) string { - backend, err := NewBackend("", "") + backend := &ecsAPIService{} + template, err := backend.convert(project) assert.NilError(t, err) - result, err := backend.Convert(project) - assert.NilError(t, err) - resultAsJSON, err := result.JSON() + resultAsJSON, err := Marshall(template) assert.NilError(t, err) return fmt.Sprintf("%s\n", string(resultAsJSON)) } @@ -351,8 +352,9 @@ func load(t *testing.T, paths ...string) *types.Project { } func convertYaml(t *testing.T, name string, yaml string) *cloudformation.Template { - model := loadConfig(t, name, yaml) - template, err := Backend{}.Convert(model) + project := loadConfig(t, name, yaml) + backend := &ecsAPIService{} + template, err := backend.convert(project) assert.NilError(t, err) return template } diff --git a/ecs/logs.go b/ecs/logs.go index ec36985f..8594e2dc 100644 --- a/ecs/logs.go +++ b/ecs/logs.go @@ -23,11 +23,12 @@ func (b *ecsAPIService) Logs(ctx context.Context, options *cli.ProjectOptions, w name = project.Name } - err := b.SDK.GetLogs(ctx, name, &logConsumer{ + consumer := logConsumer{ colors: map[string]ColorFunc{}, width: 0, writer: writer, - }) + } + err := b.SDK.GetLogs(ctx, name, consumer.Log) if err != nil { return err } diff --git a/ecs/sdk.go b/ecs/sdk.go index 1bdcbecc..5fa88937 100644 --- a/ecs/sdk.go +++ b/ecs/sdk.go @@ -3,10 +3,12 @@ package ecs import ( "context" "fmt" - "github.com/docker/api/compose" "strings" "time" + "github.com/docker/api/compose" + "github.com/docker/api/secrets" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" @@ -239,6 +241,12 @@ func (s sdk) UpdateStack(ctx context.Context, changeset string) error { return err } +const ( + StackCreate = iota + StackUpdate + StackDelete +) + func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) error { input := &cloudformation.DescribeStacksInput{ StackName: aws.String(name), @@ -298,6 +306,12 @@ func (s sdk) ListStackParameters(ctx context.Context, name string) (map[string]s return parameters, nil } +type StackResource struct { + LogicalID string + Type string + ARN string + Status string +} func (s sdk) ListStackResources(ctx context.Context, name string) ([]StackResource, error) { // FIXME handle pagination res, err := s.CF.ListStackResourcesWithContext(ctx, &cloudformation.ListStackResourcesInput{ @@ -327,7 +341,7 @@ func (s sdk) DeleteStack(ctx context.Context, name string) error { return err } -func (s sdk) CreateSecret(ctx context.Context, secret Secret) (string, error) { +func (s sdk) CreateSecret(ctx context.Context, secret secrets.Secret) (string, error) { logrus.Debug("Create secret " + secret.Name) secretStr, err := secret.GetCredString() if err != nil { @@ -345,17 +359,17 @@ func (s sdk) CreateSecret(ctx context.Context, secret Secret) (string, error) { return aws.StringValue(response.ARN), nil } -func (s sdk) InspectSecret(ctx context.Context, id string) (Secret, error) { +func (s sdk) InspectSecret(ctx context.Context, id string) (secrets.Secret, error) { logrus.Debug("Inspect secret " + id) response, err := s.SM.DescribeSecret(&secretsmanager.DescribeSecretInput{SecretId: &id}) if err != nil { - return Secret{}, err + return secrets.Secret{}, err } labels := map[string]string{} for _, tag := range response.Tags { labels[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value) } - secret := Secret{ + secret := secrets.Secret{ ID: aws.StringValue(response.ARN), Name: aws.StringValue(response.Name), Labels: labels, @@ -366,14 +380,14 @@ func (s sdk) InspectSecret(ctx context.Context, id string) (Secret, error) { return secret, nil } -func (s sdk) ListSecrets(ctx context.Context) ([]Secret, error) { +func (s sdk) ListSecrets(ctx context.Context) ([]secrets.Secret, error) { logrus.Debug("List secrets ...") response, err := s.SM.ListSecrets(&secretsmanager.ListSecretsInput{}) if err != nil { - return []Secret{}, err + return nil, err } - var secrets []Secret + var ls []secrets.Secret for _, sec := range response.SecretList { labels := map[string]string{} @@ -384,14 +398,14 @@ func (s sdk) ListSecrets(ctx context.Context) ([]Secret, error) { if sec.Description != nil { description = *sec.Description } - secrets = append(secrets, Secret{ + ls = append(ls, secrets.Secret{ ID: *sec.ARN, Name: *sec.Name, Labels: labels, Description: description, }) } - return secrets, nil + return ls, nil } func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { @@ -401,7 +415,7 @@ func (s sdk) DeleteSecret(ctx context.Context, id string, recover bool) error { return err } -func (s sdk) GetLogs(ctx context.Context, name string, consumer LogConsumer) error { +func (s sdk) GetLogs(ctx context.Context, name string, consumer func(service, container, message string)) error { logGroup := fmt.Sprintf("/docker-compose/%s", name) var startTime int64 for { @@ -424,7 +438,7 @@ func (s sdk) GetLogs(ctx context.Context, name string, consumer LogConsumer) err for _, event := range events.Events { p := strings.Split(aws.StringValue(event.LogStreamName), "/") - consumer.Log(p[1], p[2], aws.StringValue(event.Message)) + consumer(p[1], p[2], aws.StringValue(event.Message)) startTime = *event.IngestionTime } } diff --git a/ecs/secrets.go b/ecs/secrets.go index b1a4578e..80291815 100644 --- a/ecs/secrets.go +++ b/ecs/secrets.go @@ -2,20 +2,22 @@ package ecs import ( "context" + "github.com/docker/api/secrets" ) -func (b ecsAPIService) CreateSecret(ctx context.Context, secret Secret) (string, error) { + +func (b *ecsAPIService) CreateSecret(ctx context.Context, secret secrets.Secret) (string, error) { return b.SDK.CreateSecret(ctx, secret) } -func (b ecsAPIService) InspectSecret(ctx context.Context, id string) (Secret, error) { +func (b *ecsAPIService) InspectSecret(ctx context.Context, id string) (secrets.Secret, error) { return b.SDK.InspectSecret(ctx, id) } -func (b ecsAPIService) ListSecrets(ctx context.Context) ([]Secret, error) { +func (b *ecsAPIService) ListSecrets(ctx context.Context) ([]secrets.Secret, error) { return b.SDK.ListSecrets(ctx) } -func (b ecsAPIService) DeleteSecret(ctx context.Context, id string, recover bool) error { +func (b *ecsAPIService) DeleteSecret(ctx context.Context, id string, recover bool) error { return b.SDK.DeleteSecret(ctx, id, recover) } diff --git a/example/backend.go b/example/backend.go index 3206926e..2b2e9273 100644 --- a/example/backend.go +++ b/example/backend.go @@ -46,6 +46,11 @@ func (a *apiService) ComposeService() compose.Service { return &a.composeService } +func (a *apiService) SecretsService() secrets.Service { + return nil +} + + func init() { backend.Register("example", "example", service, cloud.NotImplementedCloudService) } @@ -143,6 +148,6 @@ func (cs *composeService) Logs(ctx context.Context, opts *cli.ProjectOptions, w return errdefs.ErrNotImplemented } -func Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) { +func (cs *composeService) Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) { return nil, errdefs.ErrNotImplemented } diff --git a/go.mod b/go.mod index 5dd55cc5..172569e2 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/aws/aws-sdk-go v1.34.2 github.com/awslabs/goformation/v4 v4.14.0 github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129 - github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 + github.com/compose-spec/compose-go v0.0.0-20200818070525-eb1188aae4a2 github.com/containerd/console v1.0.0 github.com/containerd/containerd v1.3.5 // indirect github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8 @@ -51,9 +51,9 @@ require ( github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 - golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 + golang.org/x/net v0.0.0-20200625001655-4c5254603344 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 // indirect google.golang.org/grpc v1.31.0 google.golang.org/protobuf v1.25.0 diff --git a/go.sum b/go.sum index 56d3d87b..32d81a88 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1 h1:F+YIkKDMHdgZBacawhFY1P9RAIgO+6uv2te6hjsjzF0= -github.com/compose-spec/compose-go v0.0.0-20200710075715-6fcc35384ee1/go.mod h1:ArodJ6gsEB7iWKrbV3fSHZ08LlBvSVB0Oqg04fX86t4= +github.com/compose-spec/compose-go v0.0.0-20200818070525-eb1188aae4a2 h1:b3JmHJVJt8zXy112yGtRq74G32sPQ8XLJxfHKaP/DOg= +github.com/compose-spec/compose-go v0.0.0-20200818070525-eb1188aae4a2/go.mod h1:P7PZ0svgjrZ8nv/XvxObbl8o0DCIE9ZbL8pllg6uL4w= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= @@ -130,6 +130,7 @@ github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8 h1:JRquW4uqIU+eSilDhuo9 github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20200708230824-53e18a9d9bfe h1:pni13lAFm1g4cjHU6c3n4qGvvJGZQK4VvKRKMseQ42E= github.com/docker/distribution v0.0.0-20200708230824-53e18a9d9bfe/go.mod h1:Oqz4IonmMNc2N7GqfTL2xkhCQx0yS6nR+HrOZJnmKIk= +github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible h1:G2hY8RD7jB9QaSmcb8mYEIg8QbEvVAB7se8+lXHZHfg= github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= @@ -222,8 +223,8 @@ github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -273,9 +274,10 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= +github.com/mjibson/esc v0.2.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= github.com/moby/term v0.0.0-20200611042045-63b9a826fb74 h1:kvRIeqJNICemq2UFLx8q/Pj+1IRNZS0XPTaMFkuNsvg= github.com/moby/term v0.0.0-20200611042045-63b9a826fb74/go.mod h1:pJ0Ot5YGdTcMdxnPMyGCfAr6fKXe0g9cDlz16MuFEBE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -394,6 +396,7 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= @@ -407,6 +410,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -414,6 +418,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -427,12 +432,15 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -443,6 +451,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -477,6 +487,10 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= diff --git a/local/backend.go b/local/backend.go index 5973062d..30a901fd 100644 --- a/local/backend.go +++ b/local/backend.go @@ -41,6 +41,7 @@ import ( "github.com/docker/api/containers" "github.com/docker/api/context/cloud" "github.com/docker/api/errdefs" + "github.com/docker/api/secrets" ) type local struct { @@ -70,6 +71,10 @@ func (ms *local) ComposeService() compose.Service { return nil } +func (ms *local) SecretsService() secrets.Service { + return nil +} + func (ms *local) Inspect(ctx context.Context, id string) (containers.Container, error) { c, err := ms.apiClient.ContainerInspect(ctx, id) if err != nil { diff --git a/ecs/types.go b/secrets/api.go similarity index 68% rename from ecs/types.go rename to secrets/api.go index f1e30073..cb52b1a0 100644 --- a/ecs/types.go +++ b/secrets/api.go @@ -1,22 +1,16 @@ -package ecs +package secrets -import "encoding/json" - -type StackResource struct { - LogicalID string - Type string - ARN string - Status string -} - -const ( - StackCreate = iota - StackUpdate - StackDelete +import ( + "context" + "encoding/json" ) -type LogConsumer interface { - Log(service, container, message string) +// Service interacts with the underlying secrets backend +type Service interface { + CreateSecret(ctx context.Context, secret Secret) (string, error) + InspectSecret(ctx context.Context, id string) (Secret, error) + ListSecrets(ctx context.Context) ([]Secret, error) + DeleteSecret(ctx context.Context, id string, recover bool) error } type Secret struct { From cfbd963c3d6912f7150fbd44654479d25ea6d434 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 18 Aug 2020 09:40:57 +0200 Subject: [PATCH 202/205] Adjust commands to latest compose-go Signed-off-by: Nicolas De Loof --- aci/backend.go | 2 +- cli/cmd/compose/compose.go | 16 ++++++++++++++++ cli/cmd/compose/down.go | 13 ++++++++----- cli/cmd/compose/logs.go | 13 ++++++++----- cli/cmd/compose/ps.go | 13 ++++++++----- cli/cmd/compose/up.go | 13 ++++++++----- ecs/backend.go | 8 ++++---- example/backend.go | 1 + 8 files changed, 54 insertions(+), 25 deletions(-) diff --git a/aci/backend.go b/aci/backend.go index 275d7d91..982a1faf 100644 --- a/aci/backend.go +++ b/aci/backend.go @@ -19,7 +19,6 @@ package aci import ( "context" "fmt" - "github.com/docker/api/secrets" "io" "net/http" "strconv" @@ -42,6 +41,7 @@ import ( "github.com/docker/api/context/cloud" "github.com/docker/api/context/store" "github.com/docker/api/errdefs" + "github.com/docker/api/secrets" ) const ( diff --git a/cli/cmd/compose/compose.go b/cli/cmd/compose/compose.go index f2afad29..b37a4ba6 100644 --- a/cli/cmd/compose/compose.go +++ b/cli/cmd/compose/compose.go @@ -18,6 +18,7 @@ package compose import ( "context" + "github.com/compose-spec/compose-go/cli" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -28,6 +29,21 @@ import ( "github.com/docker/api/errdefs" ) +type composeOptions struct { + Name string + WorkingDir string + ConfigPaths []string + Environment []string +} + +func (o *composeOptions) toProjectOptions() (*cli.ProjectOptions, error) { + return cli.NewProjectOptions(o.ConfigPaths, + cli.WithOsEnv, + cli.WithEnv(o.Environment), + cli.WithWorkingDirectory(o.WorkingDir), + cli.WithName(o.Name)) +} + // Command returns the compose command with its child commands func Command() *cobra.Command { command := &cobra.Command{ diff --git a/cli/cmd/compose/down.go b/cli/cmd/compose/down.go index 7389dadc..56052f28 100644 --- a/cli/cmd/compose/down.go +++ b/cli/cmd/compose/down.go @@ -20,18 +20,17 @@ import ( "context" "errors" - "github.com/compose-spec/compose-go/cli" "github.com/spf13/cobra" "github.com/docker/api/client" ) func downCommand() *cobra.Command { - opts := cli.ProjectOptions{} + opts := composeOptions{} downCmd := &cobra.Command{ Use: "down", RunE: func(cmd *cobra.Command, args []string) error { - return runDown(cmd.Context(), &opts) + return runDown(cmd.Context(), opts) }, } downCmd.Flags().StringVarP(&opts.Name, "project-name", "p", "", "Project name") @@ -41,7 +40,7 @@ func downCommand() *cobra.Command { return downCmd } -func runDown(ctx context.Context, opts *cli.ProjectOptions) error { +func runDown(ctx context.Context, opts composeOptions) error { c, err := client.New(ctx) if err != nil { return err @@ -52,5 +51,9 @@ func runDown(ctx context.Context, opts *cli.ProjectOptions) error { return errors.New("compose not implemented in current context") } - return composeService.Down(ctx, opts) + options, err := opts.toProjectOptions() + if err != nil { + return err + } + return composeService.Down(ctx, options) } diff --git a/cli/cmd/compose/logs.go b/cli/cmd/compose/logs.go index 84ddda0e..aa22e63d 100644 --- a/cli/cmd/compose/logs.go +++ b/cli/cmd/compose/logs.go @@ -21,18 +21,17 @@ import ( "errors" "os" - "github.com/compose-spec/compose-go/cli" "github.com/spf13/cobra" "github.com/docker/api/client" ) func logsCommand() *cobra.Command { - opts := cli.ProjectOptions{} + opts := composeOptions{} logsCmd := &cobra.Command{ Use: "logs", RunE: func(cmd *cobra.Command, args []string) error { - return runLogs(cmd.Context(), &opts) + return runLogs(cmd.Context(), opts) }, } logsCmd.Flags().StringVarP(&opts.Name, "project-name", "p", "", "Project name") @@ -42,7 +41,7 @@ func logsCommand() *cobra.Command { return logsCmd } -func runLogs(ctx context.Context, opts *cli.ProjectOptions) error { +func runLogs(ctx context.Context, opts composeOptions) error { c, err := client.New(ctx) if err != nil { return err @@ -53,5 +52,9 @@ func runLogs(ctx context.Context, opts *cli.ProjectOptions) error { return errors.New("compose not implemented in current context") } - return composeService.Logs(ctx, opts, os.Stdout) + options, err := opts.toProjectOptions() + if err != nil { + return err + } + return composeService.Logs(ctx, options, os.Stdout) } diff --git a/cli/cmd/compose/ps.go b/cli/cmd/compose/ps.go index ee152e23..109e3190 100644 --- a/cli/cmd/compose/ps.go +++ b/cli/cmd/compose/ps.go @@ -25,18 +25,17 @@ import ( "strings" "text/tabwriter" - "github.com/compose-spec/compose-go/cli" "github.com/spf13/cobra" "github.com/docker/api/client" ) func psCommand() *cobra.Command { - opts := cli.ProjectOptions{} + opts := composeOptions{} psCmd := &cobra.Command{ Use: "ps", RunE: func(cmd *cobra.Command, args []string) error { - return runPs(cmd.Context(), &opts) + return runPs(cmd.Context(), opts) }, } psCmd.Flags().StringVarP(&opts.Name, "project-name", "p", "", "Project name") @@ -46,7 +45,7 @@ func psCommand() *cobra.Command { return psCmd } -func runPs(ctx context.Context, opts *cli.ProjectOptions) error { +func runPs(ctx context.Context, opts composeOptions) error { c, err := client.New(ctx) if err != nil { return err @@ -57,7 +56,11 @@ func runPs(ctx context.Context, opts *cli.ProjectOptions) error { return errors.New("compose not implemented in current context") } - serviceList, err := composeService.Ps(ctx, opts) + options, err := opts.toProjectOptions() + if err != nil { + return err + } + serviceList, err := composeService.Ps(ctx, options) if err != nil { return err } diff --git a/cli/cmd/compose/up.go b/cli/cmd/compose/up.go index 91e7bf25..0cd28e9c 100644 --- a/cli/cmd/compose/up.go +++ b/cli/cmd/compose/up.go @@ -20,7 +20,6 @@ import ( "context" "errors" - "github.com/compose-spec/compose-go/cli" "github.com/spf13/cobra" "github.com/docker/api/client" @@ -28,11 +27,11 @@ import ( ) func upCommand() *cobra.Command { - opts := cli.ProjectOptions{} + opts := composeOptions{} upCmd := &cobra.Command{ Use: "up", RunE: func(cmd *cobra.Command, args []string) error { - return runUp(cmd.Context(), &opts) + return runUp(cmd.Context(), opts) }, } upCmd.Flags().StringVarP(&opts.Name, "project-name", "p", "", "Project name") @@ -44,7 +43,7 @@ func upCommand() *cobra.Command { return upCmd } -func runUp(ctx context.Context, opts *cli.ProjectOptions) error { +func runUp(ctx context.Context, opts composeOptions) error { c, err := client.New(ctx) if err != nil { return err @@ -56,6 +55,10 @@ func runUp(ctx context.Context, opts *cli.ProjectOptions) error { } return progress.Run(ctx, func(ctx context.Context) error { - return composeService.Up(ctx, opts) + options, err := opts.toProjectOptions() + if err != nil { + return err + } + return composeService.Up(ctx, options) }) } diff --git a/ecs/backend.go b/ecs/backend.go index a152f1a0..4997ecb9 100644 --- a/ecs/backend.go +++ b/ecs/backend.go @@ -69,16 +69,16 @@ func getEcsAPIService(ecsCtx store.EcsContext) (*ecsAPIService, error) { } return &ecsAPIService{ - ctx: ecsCtx, + ctx: ecsCtx, Region: ecsCtx.Region, - SDK: NewSDK(sess), + SDK: NewSDK(sess), }, nil } type ecsAPIService struct { - ctx store.EcsContext + ctx store.EcsContext Region string - SDK sdk + SDK sdk } func (a *ecsAPIService) ContainerService() containers.Service { diff --git a/example/backend.go b/example/backend.go index 2b2e9273..9ddd0573 100644 --- a/example/backend.go +++ b/example/backend.go @@ -31,6 +31,7 @@ import ( "github.com/docker/api/containers" "github.com/docker/api/context/cloud" "github.com/docker/api/errdefs" + "github.com/docker/api/secrets" ) type apiService struct { From 55a21122d01cb74ecfabb7486474bbb2ffb80b40 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 18 Aug 2020 11:38:23 +0200 Subject: [PATCH 203/205] Copytight header Signed-off-by: Nicolas De Loof --- compose/api.go | 7 +++---- compose/tags.go | 16 +++++++++++++++ ecs/backend.go | 2 ++ ecs/cloudformation.go | 26 +++++++++++++++++++++---- ecs/cloudformation_test.go | 21 ++++++++++++++++++-- ecs/colors.go | 16 +++++++++++++++ ecs/compatibility.go | 16 +++++++++++++++ ecs/convert.go | 40 ++++++++++++++++++-------------------- ecs/down.go | 16 +++++++++++++++ ecs/iam.go | 16 +++++++++++++++ ecs/list.go | 19 +++++++++++++++++- ecs/logs.go | 18 ++++++++++++++++- ecs/marshall.go | 16 +++++++++++++++ ecs/sdk.go | 25 ++++++++++++++++++++---- ecs/secrets.go | 18 ++++++++++++++++- ecs/secrets/Dockerfile | 14 +++++++++++++ ecs/secrets/init.go | 16 +++++++++++++++ ecs/secrets/init_test.go | 37 ++++++++++++++++++++++++++--------- ecs/secrets/main/main.go | 19 +++++++++++++++++- ecs/up.go | 18 ++++++++++++++++- ecs/wait.go | 19 +++++++++++++++++- ecs/x.go | 16 +++++++++++++++ go.sum | 5 ----- secrets/api.go | 16 +++++++++++++++ 24 files changed, 377 insertions(+), 55 deletions(-) diff --git a/compose/api.go b/compose/api.go index ec055e7e..2cd4c3a1 100644 --- a/compose/api.go +++ b/compose/api.go @@ -37,8 +37,7 @@ type Service interface { Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) } - -type LoadBalancer struct { +type PortPublisher struct { URL string TargetPort int PublishedPort int @@ -51,5 +50,5 @@ type ServiceStatus struct { Replicas int Desired int Ports []string - LoadBalancers []LoadBalancer -} \ No newline at end of file + LoadBalancers []PortPublisher +} diff --git a/compose/tags.go b/compose/tags.go index 43236d45..cd9ecbe3 100644 --- a/compose/tags.go +++ b/compose/tags.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package compose const ( diff --git a/ecs/backend.go b/ecs/backend.go index 156289e7..be9c68f9 100644 --- a/ecs/backend.go +++ b/ecs/backend.go @@ -18,8 +18,10 @@ package ecs import ( "context" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" + "github.com/docker/api/secrets" "github.com/docker/api/backend" diff --git a/ecs/cloudformation.go b/ecs/cloudformation.go index 85f40a6a..0e9685d3 100644 --- a/ecs/cloudformation.go +++ b/ecs/cloudformation.go @@ -1,14 +1,32 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( "context" "fmt" - "github.com/compose-spec/compose-go/cli" - "github.com/docker/api/compose" "io/ioutil" "regexp" "strings" + "github.com/compose-spec/compose-go/cli" + + "github.com/docker/api/compose" + ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/elbv2" cloudmapapi "github.com/aws/aws-sdk-go/service/servicediscovery" @@ -329,13 +347,13 @@ func createLoadBalancer(project *types.Project, template *cloudformation.Templat } if ports == 0 { // Project do not expose any port (batch jobs?) - // So no need to create a LoadBalancer + // So no need to create a PortPublisher return "" } // load balancer names are limited to 32 characters total loadBalancerName := fmt.Sprintf("%.32s", fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name))) - // Create LoadBalancer if `ParameterLoadBalancerName` is not set + // Create PortPublisher if `ParameterLoadBalancerName` is not set template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) loadBalancerType := getLoadBalancerType(project) diff --git a/ecs/cloudformation_test.go b/ecs/cloudformation_test.go index 3943ee1f..5f1d6ba4 100644 --- a/ecs/cloudformation_test.go +++ b/ecs/cloudformation_test.go @@ -1,11 +1,28 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( "fmt" - "github.com/docker/api/compose" "reflect" "testing" + "github.com/docker/api/compose" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ec2" @@ -187,7 +204,7 @@ services: for _, r := range template.Resources { assert.Check(t, r.AWSCloudFormationType() != "AWS::ElasticLoadBalancingV2::TargetGroup") assert.Check(t, r.AWSCloudFormationType() != "AWS::ElasticLoadBalancingV2::Listener") - assert.Check(t, r.AWSCloudFormationType() != "AWS::ElasticLoadBalancingV2::LoadBalancer") + assert.Check(t, r.AWSCloudFormationType() != "AWS::ElasticLoadBalancingV2::PortPublisher") } } diff --git a/ecs/colors.go b/ecs/colors.go index 3af58af5..22d609c8 100644 --- a/ecs/colors.go +++ b/ecs/colors.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( diff --git a/ecs/compatibility.go b/ecs/compatibility.go index 2fdd527a..316c6b88 100644 --- a/ecs/compatibility.go +++ b/ecs/compatibility.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( diff --git a/ecs/convert.go b/ecs/convert.go index 17db6064..477989cd 100644 --- a/ecs/convert.go +++ b/ecs/convert.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( @@ -14,11 +30,11 @@ import ( ecsapi "github.com/aws/aws-sdk-go/service/ecs" "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/ecs" - "github.com/awslabs/goformation/v4/cloudformation/tags" "github.com/compose-spec/compose-go/types" - "github.com/docker/api/ecs/secrets" "github.com/docker/cli/opts" "github.com/joho/godotenv" + + "github.com/docker/api/ecs/secrets" ) const secretsInitContainerImage = "docker/ecs-secrets-sidecar" @@ -185,7 +201,7 @@ func createEnvironment(project *types.Project, service types.ServiceConfig) ([]e if err != nil { return nil, err } - defer file.Close() + defer file.Close() // nolint:errcheck env, err := godotenv.Parse(file) if err != nil { @@ -234,17 +250,6 @@ func getLogConfiguration(service types.ServiceConfig, project *types.Project) *e return logConfiguration } -func toTags(labels types.Labels) []tags.Tag { - t := []tags.Tag{} - for n, v := range labels { - t = append(t, tags.Tag{ - Key: n, - Value: v, - }) - } - return t -} - func toSystemControls(sysctls types.Mapping) []ecs.TaskDefinition_SystemControl { sys := []ecs.TaskDefinition_SystemControl{} for k, v := range sysctls { @@ -324,13 +329,6 @@ func toContainerReservation(service types.ServiceConfig) (string, int, error) { return reservations.NanoCPUs, int(reservations.MemoryBytes / MiB), nil } -func toRequiresCompatibilities(isolation string) []*string { - if isolation == "" { - return nil - } - return []*string{&isolation} -} - func toPlacementConstraints(deploy *types.DeployConfig) []ecs.TaskDefinition_TaskDefinitionPlacementConstraint { if deploy == nil || deploy.Placement.Constraints == nil || len(deploy.Placement.Constraints) == 0 { return nil diff --git a/ecs/down.go b/ecs/down.go index 849b1c67..55e0dec9 100644 --- a/ecs/down.go +++ b/ecs/down.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( diff --git a/ecs/iam.go b/ecs/iam.go index 3dca431b..30670b0f 100644 --- a/ecs/iam.go +++ b/ecs/iam.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs const ( diff --git a/ecs/list.go b/ecs/list.go index 7642471b..02be946e 100644 --- a/ecs/list.go +++ b/ecs/list.go @@ -1,11 +1,28 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( "context" "fmt" - "github.com/docker/api/compose" "strings" + "github.com/docker/api/compose" + "github.com/compose-spec/compose-go/cli" ) diff --git a/ecs/logs.go b/ecs/logs.go index 8594e2dc..58010fc1 100644 --- a/ecs/logs.go +++ b/ecs/logs.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( @@ -50,7 +66,7 @@ func (l *logConsumer) Log(service, container, message string) { for _, line := range strings.Split(message, "\n") { buf := bytes.NewBufferString(fmt.Sprintf("%s %s\n", cf(prefix), line)) - l.writer.Write(buf.Bytes()) + l.writer.Write(buf.Bytes()) // nolint:errcheck } } diff --git a/ecs/marshall.go b/ecs/marshall.go index 41456362..8ef38a9d 100644 --- a/ecs/marshall.go +++ b/ecs/marshall.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( diff --git a/ecs/sdk.go b/ecs/sdk.go index 5fa88937..3d499b1b 100644 --- a/ecs/sdk.go +++ b/ecs/sdk.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( @@ -312,6 +328,7 @@ type StackResource struct { ARN string Status string } + func (s sdk) ListStackResources(ctx context.Context, name string) ([]StackResource, error) { // FIXME handle pagination res, err := s.CF.ListStackResourcesWithContext(ctx, &cloudformation.ListStackResourcesInput{ @@ -488,7 +505,7 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, arns []string return status, nil } -func (s sdk) getURLWithPortMapping(ctx context.Context, targetGroupArns []string) ([]compose.LoadBalancer, error) { +func (s sdk) getURLWithPortMapping(ctx context.Context, targetGroupArns []string) ([]compose.PortPublisher, error) { if len(targetGroupArns) == 0 { return nil, nil } @@ -522,14 +539,14 @@ func (s sdk) getURLWithPortMapping(ctx context.Context, targetGroupArns []string } return nil } - loadBalancers := []compose.LoadBalancer{} + loadBalancers := []compose.PortPublisher{} for _, tg := range groups.TargetGroups { for _, lbarn := range tg.LoadBalancerArns { lb := filterLB(lbarn, lbs.LoadBalancers) if lb == nil { continue } - loadBalancers = append(loadBalancers, compose.LoadBalancer{ + loadBalancers = append(loadBalancers, compose.PortPublisher{ URL: aws.StringValue(lb.DNSName), TargetPort: int(aws.Int64Value(tg.Port)), PublishedPort: int(aws.Int64Value(tg.Port)), @@ -573,7 +590,7 @@ func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string } func (s sdk) LoadBalancerExists(ctx context.Context, arn string) (bool, error) { - logrus.Debug("CheckRequirements if LoadBalancer exists: ", arn) + logrus.Debug("CheckRequirements if PortPublisher exists: ", arn) lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{ LoadBalancerArns: []*string{aws.String(arn)}, }) diff --git a/ecs/secrets.go b/ecs/secrets.go index 80291815..60c8964f 100644 --- a/ecs/secrets.go +++ b/ecs/secrets.go @@ -1,11 +1,27 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( "context" + "github.com/docker/api/secrets" ) - func (b *ecsAPIService) CreateSecret(ctx context.Context, secret secrets.Secret) (string, error) { return b.SDK.CreateSecret(ctx, secret) } diff --git a/ecs/secrets/Dockerfile b/ecs/secrets/Dockerfile index 238f43dd..5490ffc4 100644 --- a/ecs/secrets/Dockerfile +++ b/ecs/secrets/Dockerfile @@ -1,3 +1,17 @@ +# Copyright 2020 Docker, Inc. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM golang:1.14.4-alpine AS builder WORKDIR $GOPATH/src/github.com/docker/api/ecs/secrets COPY . . diff --git a/ecs/secrets/init.go b/ecs/secrets/init.go index 1903d3cc..8ee28f29 100644 --- a/ecs/secrets/init.go +++ b/ecs/secrets/init.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package secrets import ( diff --git a/ecs/secrets/init_test.go b/ecs/secrets/init_test.go index dd068f5a..5389a5c5 100644 --- a/ecs/secrets/init_test.go +++ b/ecs/secrets/init_test.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package secrets import ( @@ -12,10 +28,11 @@ import ( func TestRawSecret(t *testing.T) { dir := fs.NewDir(t, "secrets").Path() - os.Setenv("raw", "something_secret") - defer os.Unsetenv("raw") + err := os.Setenv("raw", "something_secret") + assert.NilError(t, err) + defer os.Unsetenv("raw") // nolint:errcheck - err := CreateSecretFiles(Secret{ + err = CreateSecretFiles(Secret{ Name: "raw", Keys: nil, }, dir) @@ -28,14 +45,15 @@ func TestRawSecret(t *testing.T) { func TestSelectedKeysSecret(t *testing.T) { dir := fs.NewDir(t, "secrets").Path() - os.Setenv("json", ` + err := os.Setenv("json", ` { "foo": "bar", "zot": "qix" }`) - defer os.Unsetenv("json") + assert.NilError(t, err) + defer os.Unsetenv("json") // nolint:errcheck - err := CreateSecretFiles(Secret{ + err = CreateSecretFiles(Secret{ Name: "json", Keys: []string{"foo"}, }, dir) @@ -51,14 +69,15 @@ func TestSelectedKeysSecret(t *testing.T) { func TestAllKeysSecret(t *testing.T) { dir := fs.NewDir(t, "secrets").Path() - os.Setenv("json", ` + err := os.Setenv("json", ` { "foo": "bar", "zot": "qix" }`) - defer os.Unsetenv("json") + assert.NilError(t, err) + defer os.Unsetenv("json") // nolint:errcheck - err := CreateSecretFiles(Secret{ + err = CreateSecretFiles(Secret{ Name: "json", Keys: []string{"*"}, }, dir) diff --git a/ecs/secrets/main/main.go b/ecs/secrets/main/main.go index c66c6de3..b70baf0d 100644 --- a/ecs/secrets/main/main.go +++ b/ecs/secrets/main/main.go @@ -1,10 +1,27 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package main import ( "encoding/json" "fmt" - "github.com/docker/api/ecs/secrets" "os" + + "github.com/docker/api/ecs/secrets" ) const secretsFolder = "/run/secrets" diff --git a/ecs/up.go b/ecs/up.go index 44d16227..a3e0b4dc 100644 --- a/ecs/up.go +++ b/ecs/up.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( @@ -85,7 +101,7 @@ func (b *ecsAPIService) Up(ctx context.Context, options *cli.ProjectOptions) err go func() { <-signalChan fmt.Println("user interrupted deployment. Deleting stack...") - b.Down(ctx, options) + b.Down(ctx, options) // nolint:errcheck }() err = b.WaitStackCompletion(ctx, project.Name, operation) diff --git a/ecs/wait.go b/ecs/wait.go index 4ca41d85..04632ee6 100644 --- a/ecs/wait.go +++ b/ecs/wait.go @@ -1,13 +1,30 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs import ( "context" "fmt" - "github.com/docker/api/progress" "sort" "strings" "time" + "github.com/docker/api/progress" + "github.com/aws/aws-sdk-go/aws" ) diff --git a/ecs/x.go b/ecs/x.go index 9dee6a36..1f2ba5ce 100644 --- a/ecs/x.go +++ b/ecs/x.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package ecs const ( diff --git a/go.sum b/go.sum index 2afc5e41..9312e8a0 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.34.2 h1:9vCknCdTAmmV4ht7lPuda7aJXzllXwEQyCMZKJHjBrM= -github.com/aws/aws-sdk-go v1.34.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.6 h1:2aPXQGkR6xeheN5dns13mSoDWeUlj4wDmfZ+8ZDHauw= github.com/aws/aws-sdk-go v1.34.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/awslabs/goformation/v4 v4.14.0 h1:E2Pet9eIqA4qzt3dzzzE4YN83V4Kyfbcio0VokBC9TA= @@ -132,7 +130,6 @@ github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8 h1:JRquW4uqIU+eSilDhuo9 github.com/docker/cli v0.0.0-20200528204125-dd360c7c0de8/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20200708230824-53e18a9d9bfe h1:pni13lAFm1g4cjHU6c3n4qGvvJGZQK4VvKRKMseQ42E= github.com/docker/distribution v0.0.0-20200708230824-53e18a9d9bfe/go.mod h1:Oqz4IonmMNc2N7GqfTL2xkhCQx0yS6nR+HrOZJnmKIk= -github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible h1:G2hY8RD7jB9QaSmcb8mYEIg8QbEvVAB7se8+lXHZHfg= github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= @@ -534,8 +531,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.58.0 h1:VdDvTzv/005R8vEFyQ56bpEnOKTNPbpJhL0VCohxlQw= -gopkg.in/ini.v1 v1.58.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.60.0 h1:P5ZzC7RJO04094NJYlEnBdFK2wwmnCAy/+7sAzvWs60= gopkg.in/ini.v1 v1.60.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/secrets/api.go b/secrets/api.go index cb52b1a0..f1231b7d 100644 --- a/secrets/api.go +++ b/secrets/api.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package secrets import ( From 0a6d7b1c6d25a3330015ed060a94e2028f9b2279 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 18 Aug 2020 16:25:26 +0200 Subject: [PATCH 204/205] Reduce cyclomatic complexity Signed-off-by: Nicolas De Loof --- ecs/convert.go | 136 +++++++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 61 deletions(-) diff --git a/ecs/convert.go b/ecs/convert.go index 477989cd..06557210 100644 --- a/ecs/convert.go +++ b/ecs/convert.go @@ -62,72 +62,25 @@ func convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi logConfiguration := getLogConfiguration(service, project) var ( - containers []ecs.TaskDefinition_ContainerDefinition + initContainers []ecs.TaskDefinition_ContainerDefinition volumes []ecs.TaskDefinition_Volume mounts []ecs.TaskDefinition_MountPoint - initContainers []ecs.TaskDefinition_ContainerDependency ) if len(service.Secrets) > 0 { - initContainerName := fmt.Sprintf("%s_Secrets_InitContainer", normalizeResourceName(service.Name)) - volumes = append(volumes, ecs.TaskDefinition_Volume{ - Name: "secrets", - }) - mounts = append(mounts, ecs.TaskDefinition_MountPoint{ - ContainerPath: "/run/secrets/", - ReadOnly: true, - SourceVolume: "secrets", - }) - initContainers = append(initContainers, ecs.TaskDefinition_ContainerDependency{ - Condition: ecsapi.ContainerConditionSuccess, - ContainerName: initContainerName, - }) - - var ( - args []secrets.Secret - taskSecrets []ecs.TaskDefinition_Secret - ) - for _, s := range service.Secrets { - secretConfig := project.Secrets[s.Source] - if s.Target == "" { - s.Target = s.Source - } - taskSecrets = append(taskSecrets, ecs.TaskDefinition_Secret{ - Name: s.Target, - ValueFrom: secretConfig.Name, - }) - var keys []string - if ext, ok := secretConfig.Extensions[ExtensionKeys]; ok { - if key, ok := ext.(string); ok { - keys = append(keys, key) - } else { - for _, k := range ext.([]interface{}) { - keys = append(keys, k.(string)) - } - } - } - args = append(args, secrets.Secret{ - Name: s.Target, - Keys: keys, - }) - } - command, err := json.Marshal(args) + secretsVolume, secretsMount, secretsSideCar, err := createSecretsSideCar(project, service, logConfiguration) if err != nil { return nil, err } - containers = append(containers, ecs.TaskDefinition_ContainerDefinition{ - Name: initContainerName, - Image: secretsInitContainerImage, - Command: []string{string(command)}, - Essential: false, // FIXME this will be ignored, see https://github.com/awslabs/goformation/issues/61#issuecomment-625139607 - LogConfiguration: logConfiguration, - MountPoints: []ecs.TaskDefinition_MountPoint{ - { - ContainerPath: "/run/secrets/", - ReadOnly: false, - SourceVolume: "secrets", - }, - }, - Secrets: taskSecrets, + initContainers = append(initContainers, secretsSideCar) + volumes = append(volumes, secretsVolume) + mounts = append(mounts, secretsMount) + } + + var dependencies []ecs.TaskDefinition_ContainerDependency + for _, c := range initContainers { + dependencies = append(dependencies, ecs.TaskDefinition_ContainerDependency{ + Condition: ecsapi.ContainerConditionSuccess, + ContainerName: c.Name, }) } @@ -136,10 +89,10 @@ func convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi return nil, err } - containers = append(containers, ecs.TaskDefinition_ContainerDefinition{ + containers := append(initContainers, ecs.TaskDefinition_ContainerDefinition{ Command: service.Command, DisableNetworking: service.NetworkMode == "none", - DependsOnProp: initContainers, + DependsOnProp: dependencies, DnsSearchDomains: service.DNSSearch, DnsServers: service.DNS, DockerSecurityOptions: service.SecurityOpt, @@ -188,6 +141,67 @@ func convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi }, nil } +func createSecretsSideCar(project *types.Project, service types.ServiceConfig, logConfiguration *ecs.TaskDefinition_LogConfiguration) (ecs.TaskDefinition_Volume, ecs.TaskDefinition_MountPoint, ecs.TaskDefinition_ContainerDefinition, error) { + initContainerName := fmt.Sprintf("%s_Secrets_InitContainer", normalizeResourceName(service.Name)) + secretsVolume := ecs.TaskDefinition_Volume{ + Name: "secrets", + } + secretsMount := ecs.TaskDefinition_MountPoint{ + ContainerPath: "/run/secrets/", + ReadOnly: true, + SourceVolume: "secrets", + } + + var ( + args []secrets.Secret + taskSecrets []ecs.TaskDefinition_Secret + ) + for _, s := range service.Secrets { + secretConfig := project.Secrets[s.Source] + if s.Target == "" { + s.Target = s.Source + } + taskSecrets = append(taskSecrets, ecs.TaskDefinition_Secret{ + Name: s.Target, + ValueFrom: secretConfig.Name, + }) + var keys []string + if ext, ok := secretConfig.Extensions[ExtensionKeys]; ok { + if key, ok := ext.(string); ok { + keys = append(keys, key) + } else { + for _, k := range ext.([]interface{}) { + keys = append(keys, k.(string)) + } + } + } + args = append(args, secrets.Secret{ + Name: s.Target, + Keys: keys, + }) + } + command, err := json.Marshal(args) + if err != nil { + return ecs.TaskDefinition_Volume{}, ecs.TaskDefinition_MountPoint{}, ecs.TaskDefinition_ContainerDefinition{}, err + } + secretsSideCar := ecs.TaskDefinition_ContainerDefinition{ + Name: initContainerName, + Image: secretsInitContainerImage, + Command: []string{string(command)}, + Essential: false, // FIXME this will be ignored, see https://github.com/awslabs/goformation/issues/61#issuecomment-625139607 + LogConfiguration: logConfiguration, + MountPoints: []ecs.TaskDefinition_MountPoint{ + { + ContainerPath: "/run/secrets/", + ReadOnly: false, + SourceVolume: "secrets", + }, + }, + Secrets: taskSecrets, + } + return secretsVolume, secretsMount, secretsSideCar, nil +} + func createEnvironment(project *types.Project, service types.ServiceConfig) ([]ecs.TaskDefinition_KeyValuePair, error) { environment := map[string]*string{} for _, f := range service.EnvFile { From ec4615ae578ab6e7836434a422474e043ed6baea Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 18 Aug 2020 16:56:42 +0200 Subject: [PATCH 205/205] Apply linter recommendations Signed-off-by: Nicolas De Loof --- aci/backend.go | 1 - backend/backend.go | 3 +- cli/cmd/compose/compose.go | 2 +- client/client.go | 6 +-- compose/api.go | 14 ++++--- compose/tags.go | 3 ++ ecs/backend.go | 2 +- ecs/cloudformation.go | 79 ++++++++++++++++++-------------------- ecs/cloudformation_test.go | 73 +++++++++++++++++++---------------- ecs/colors.go | 48 ++++++++++------------- ecs/compatibility.go | 10 ++--- ecs/context.go | 2 +- ecs/convert.go | 29 +++++++------- ecs/down.go | 2 +- ecs/iam.go | 13 ++++--- ecs/list.go | 4 +- ecs/logs.go | 6 +-- ecs/marshall.go | 2 +- ecs/sdk.go | 52 ++++++++++++------------- ecs/secrets/init.go | 2 + ecs/secrets/main/main.go | 6 +-- ecs/up.go | 20 +++++----- ecs/wait.go | 10 ++--- ecs/x.go | 22 +++++------ secrets/api.go | 4 ++ 25 files changed, 213 insertions(+), 202 deletions(-) diff --git a/aci/backend.go b/aci/backend.go index 982a1faf..3c6fd17b 100644 --- a/aci/backend.go +++ b/aci/backend.go @@ -132,7 +132,6 @@ func (a *aciAPIService) SecretsService() secrets.Service { return nil } - type aciContainerService struct { ctx store.AciContext } diff --git a/backend/backend.go b/backend/backend.go index b5d5f9d5..e44b218b 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -20,12 +20,13 @@ import ( "context" "errors" "fmt" - "github.com/docker/api/secrets" + "github.com/sirupsen/logrus" "github.com/docker/api/compose" "github.com/docker/api/containers" "github.com/docker/api/context/cloud" + "github.com/docker/api/secrets" ) var ( diff --git a/cli/cmd/compose/compose.go b/cli/cmd/compose/compose.go index b37a4ba6..1e2e52e4 100644 --- a/cli/cmd/compose/compose.go +++ b/cli/cmd/compose/compose.go @@ -18,8 +18,8 @@ package compose import ( "context" - "github.com/compose-spec/compose-go/cli" + "github.com/compose-spec/compose-go/cli" "github.com/pkg/errors" "github.com/spf13/cobra" diff --git a/client/client.go b/client/client.go index 86230505..64b98e89 100644 --- a/client/client.go +++ b/client/client.go @@ -18,14 +18,14 @@ package client import ( "context" - "github.com/docker/api/secrets" - "github.com/docker/api/context/cloud" + "github.com/docker/api/secrets" "github.com/docker/api/backend" "github.com/docker/api/compose" "github.com/docker/api/containers" apicontext "github.com/docker/api/context" + "github.com/docker/api/context/cloud" "github.com/docker/api/context/store" ) @@ -71,7 +71,7 @@ func (c *Client) ComposeService() compose.Service { return c.bs.ComposeService() } -// ComposeService returns the backend service for the current context +// SecretsService returns the backend service for the current context func (c *Client) SecretsService() secrets.Service { return c.bs.SecretsService() } diff --git a/compose/api.go b/compose/api.go index 2cd4c3a1..5a917037 100644 --- a/compose/api.go +++ b/compose/api.go @@ -37,6 +37,7 @@ type Service interface { Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) } +// PortPublisher hold status about published port type PortPublisher struct { URL string TargetPort int @@ -44,11 +45,12 @@ type PortPublisher struct { Protocol string } +// ServiceStatus hold status about a service type ServiceStatus struct { - ID string - Name string - Replicas int - Desired int - Ports []string - LoadBalancers []PortPublisher + ID string + Name string + Replicas int + Desired int + Ports []string + Publishers []PortPublisher } diff --git a/compose/tags.go b/compose/tags.go index cd9ecbe3..300e55bb 100644 --- a/compose/tags.go +++ b/compose/tags.go @@ -17,7 +17,10 @@ package compose const ( + // ProjectTag allow to track resource related to a compose project ProjectTag = "com.docker.compose.project" + // NetworkTag allow to track resource related to a compose network NetworkTag = "com.docker.compose.network" + // ServiceTag allow to track resource related to a compose service ServiceTag = "com.docker.compose.service" ) diff --git a/ecs/backend.go b/ecs/backend.go index be9c68f9..107902f3 100644 --- a/ecs/backend.go +++ b/ecs/backend.go @@ -76,7 +76,7 @@ func getEcsAPIService(ecsCtx store.EcsContext) (*ecsAPIService, error) { return &ecsAPIService{ ctx: ecsCtx, Region: ecsCtx.Region, - SDK: NewSDK(sess), + SDK: newSDK(sess), }, nil } diff --git a/ecs/cloudformation.go b/ecs/cloudformation.go index 0e9685d3..9481a350 100644 --- a/ecs/cloudformation.go +++ b/ecs/cloudformation.go @@ -46,11 +46,11 @@ import ( ) const ( - ParameterClusterName = "ParameterClusterName" - ParameterVPCId = "ParameterVPCId" - ParameterSubnet1Id = "ParameterSubnet1Id" - ParameterSubnet2Id = "ParameterSubnet2Id" - ParameterLoadBalancerARN = "ParameterLoadBalancerARN" + parameterClusterName = "ParameterClusterName" + parameterVPCId = "ParameterVPCId" + parameterSubnet1Id = "ParameterSubnet1Id" + parameterSubnet2Id = "ParameterSubnet2Id" + parameterLoadBalancerARN = "ParameterLoadBalancerARN" ) func (b *ecsAPIService) Convert(ctx context.Context, opts *cli.ProjectOptions) ([]byte, error) { @@ -62,12 +62,12 @@ func (b *ecsAPIService) Convert(ctx context.Context, opts *cli.ProjectOptions) ( if err != nil { return nil, err } - return Marshall(template) + return marshall(template) } // Convert a compose project into a CloudFormation template -func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Template, error) { - var checker compatibility.Checker = &FargateCompatibilityChecker{ +func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Template, error) { //nolint:gocyclo + var checker compatibility.Checker = &fargateCompatibilityChecker{ compatibility.AllowList{ Supported: compatibleComposeAttributes, }, @@ -86,12 +86,12 @@ func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Templat template := cloudformation.NewTemplate() template.Description = "CloudFormation template created by Docker for deploying applications on Amazon ECS" - template.Parameters[ParameterClusterName] = cloudformation.Parameter{ + template.Parameters[parameterClusterName] = cloudformation.Parameter{ Type: "String", Description: "Name of the ECS cluster to deploy to (optional)", } - template.Parameters[ParameterVPCId] = cloudformation.Parameter{ + template.Parameters[parameterVPCId] = cloudformation.Parameter{ Type: "AWS::EC2::VPC::Id", Description: "ID of the VPC", } @@ -103,28 +103,28 @@ func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Templat Description: "The list of SubnetIds, for at least two Availability Zones in the region in your VPC", } */ - template.Parameters[ParameterSubnet1Id] = cloudformation.Parameter{ + template.Parameters[parameterSubnet1Id] = cloudformation.Parameter{ Type: "AWS::EC2::Subnet::Id", Description: "SubnetId, for Availability Zone 1 in the region in your VPC", } - template.Parameters[ParameterSubnet2Id] = cloudformation.Parameter{ + template.Parameters[parameterSubnet2Id] = cloudformation.Parameter{ Type: "AWS::EC2::Subnet::Id", Description: "SubnetId, for Availability Zone 2 in the region in your VPC", } - template.Parameters[ParameterLoadBalancerARN] = cloudformation.Parameter{ + template.Parameters[parameterLoadBalancerARN] = cloudformation.Parameter{ Type: "String", Description: "Name of the LoadBalancer to connect to (optional)", } // Create Cluster is `ParameterClusterName` parameter is not set - template.Conditions["CreateCluster"] = cloudformation.Equals("", cloudformation.Ref(ParameterClusterName)) + template.Conditions["CreateCluster"] = cloudformation.Equals("", cloudformation.Ref(parameterClusterName)) cluster := createCluster(project, template) networks := map[string]string{} for _, net := range project.Networks { - networks[net.Name] = convertNetwork(project, net, cloudformation.Ref(ParameterVPCId), template) + networks[net.Name] = convertNetwork(project, net, cloudformation.Ref(parameterVPCId), template) } for i, s := range project.Secrets { @@ -175,9 +175,6 @@ func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Templat template.Resources[taskDefinition] = definition var healthCheck *cloudmap.Service_HealthCheckConfig - if service.HealthCheck != nil && !service.HealthCheck.Disable { - // FIXME ECS only support HTTP(s) health checks, while Docker only support CMD - } serviceRegistry := createServiceRegistry(service, template, healthCheck) @@ -242,8 +239,8 @@ func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Templat AssignPublicIp: ecsapi.AssignPublicIpEnabled, SecurityGroups: serviceSecurityGroups, Subnets: []string{ - cloudformation.Ref(ParameterSubnet1Id), - cloudformation.Ref(ParameterSubnet2Id), + cloudformation.Ref(parameterSubnet1Id), + cloudformation.Ref(parameterSubnet2Id), }, }, }, @@ -268,7 +265,7 @@ func (b *ecsAPIService) convert(project *types.Project) (*cloudformation.Templat func createLogGroup(project *types.Project, template *cloudformation.Template) { retention := 0 - if v, ok := project.Extensions[ExtensionRetention]; ok { + if v, ok := project.Extensions[extensionRetention]; ok { retention = v.(int) } logGroup := fmt.Sprintf("/docker-compose/%s", project.Name) @@ -285,11 +282,11 @@ func computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) { return minPercent, maxPercent, nil } updateConfig := service.Deploy.UpdateConfig - min, okMin := updateConfig.Extensions[ExtensionMinPercent] + min, okMin := updateConfig.Extensions[extensionMinPercent] if okMin { minPercent = min.(int) } - max, okMax := updateConfig.Extensions[ExtensionMaxPercent] + max, okMax := updateConfig.Extensions[extensionMaxPercent] if okMax { maxPercent = max.(int) } @@ -333,7 +330,7 @@ func getLoadBalancerSecurityGroups(project *types.Project, template *cloudformat securityGroups := []string{} for _, network := range project.Networks { if !network.Internal { - net := convertNetwork(project, network, cloudformation.Ref(ParameterVPCId), template) + net := convertNetwork(project, network, cloudformation.Ref(parameterVPCId), template) securityGroups = append(securityGroups, net) } } @@ -354,7 +351,7 @@ func createLoadBalancer(project *types.Project, template *cloudformation.Templat // load balancer names are limited to 32 characters total loadBalancerName := fmt.Sprintf("%.32s", fmt.Sprintf("%sLoadBalancer", strings.Title(project.Name))) // Create PortPublisher if `ParameterLoadBalancerName` is not set - template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(ParameterLoadBalancerARN)) + template.Conditions["CreateLoadBalancer"] = cloudformation.Equals("", cloudformation.Ref(parameterLoadBalancerARN)) loadBalancerType := getLoadBalancerType(project) securityGroups := []string{} @@ -367,8 +364,8 @@ func createLoadBalancer(project *types.Project, template *cloudformation.Templat Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, SecurityGroups: securityGroups, Subnets: []string{ - cloudformation.Ref(ParameterSubnet1Id), - cloudformation.Ref(ParameterSubnet2Id), + cloudformation.Ref(parameterSubnet1Id), + cloudformation.Ref(parameterSubnet2Id), }, Tags: []tags.Tag{ { @@ -379,7 +376,7 @@ func createLoadBalancer(project *types.Project, template *cloudformation.Templat Type: loadBalancerType, AWSCloudFormationCondition: "CreateLoadBalancer", } - return cloudformation.If("CreateLoadBalancer", cloudformation.Ref(loadBalancerName), cloudformation.Ref(ParameterLoadBalancerARN)) + return cloudformation.If("CreateLoadBalancer", cloudformation.Ref(loadBalancerName), cloudformation.Ref(parameterLoadBalancerARN)) } func createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, targetGroupName string, loadBalancerARN string, protocol string) string { @@ -427,7 +424,7 @@ func createTargetGroup(project *types.Project, service types.ServiceConfig, port Value: project.Name, }, }, - VpcId: cloudformation.Ref(ParameterVPCId), + VpcId: cloudformation.Ref(parameterVPCId), TargetType: elbv2.TargetTypeEnumIp, } return targetGroupName @@ -462,7 +459,7 @@ func createServiceRegistry(service types.ServiceConfig, template *cloudformation func createTaskExecutionRole(service types.ServiceConfig, err error, definition *ecs.TaskDefinition, template *cloudformation.Template) (string, error) { taskExecutionRole := fmt.Sprintf("%sTaskExecutionRole", normalizeResourceName(service.Name)) - policy, err := getPolicy(definition) + policy := getPolicy(definition) if err != nil { return taskExecutionRole, err } @@ -474,16 +471,16 @@ func createTaskExecutionRole(service types.ServiceConfig, err error, definition }) } - if roles, ok := service.Extensions[ExtensionRole]; ok { + if roles, ok := service.Extensions[extensionRole]; ok { rolePolicies = append(rolePolicies, iam.Role_Policy{ PolicyDocument: roles, }) } managedPolicies := []string{ - ECSTaskExecutionPolicy, - ECRReadOnlyPolicy, + ecsTaskExecutionPolicy, + ecrReadOnlyPolicy, } - if v, ok := service.Extensions[ExtensionManagedPolicies]; ok { + if v, ok := service.Extensions[extensionManagedPolicies]; ok { for _, s := range v.([]interface{}) { managedPolicies = append(managedPolicies, s.(string)) } @@ -507,7 +504,7 @@ func createCluster(project *types.Project, template *cloudformation.Template) st }, AWSCloudFormationCondition: "CreateCluster", } - cluster := cloudformation.If("CreateCluster", cloudformation.Ref("Cluster"), cloudformation.Ref(ParameterClusterName)) + cluster := cloudformation.If("CreateCluster", cloudformation.Ref("Cluster"), cloudformation.Ref(parameterClusterName)) return cluster } @@ -515,12 +512,12 @@ func createCloudMap(project *types.Project, template *cloudformation.Template) { template.Resources["CloudMap"] = &cloudmap.PrivateDnsNamespace{ Description: fmt.Sprintf("Service Map for Docker Compose project %s", project.Name), Name: fmt.Sprintf("%s.local", project.Name), - Vpc: cloudformation.Ref(ParameterVPCId), + Vpc: cloudformation.Ref(parameterVPCId), } } func convertNetwork(project *types.Project, net types.NetworkConfig, vpc string, template *cloudformation.Template) string { - if sg, ok := net.Extensions[ExtensionSecurityGroup]; ok { + if sg, ok := net.Extensions[extensionSecurityGroup]; ok { logrus.Debugf("Security Group for network %q set by user to %q", net.Name, sg) return sg.(string) } @@ -583,7 +580,7 @@ func normalizeResourceName(s string) string { return strings.Title(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(s, "")) } -func getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { +func getPolicy(taskDef *ecs.TaskDefinition) *PolicyDocument { arns := []string{} for _, container := range taskDef.ContainerDefinitions { if container.RepositoryCredentials != nil { @@ -601,12 +598,12 @@ func getPolicy(taskDef *ecs.TaskDefinition) (*PolicyDocument, error) { Statement: []PolicyStatement{ { Effect: "Allow", - Action: []string{ActionGetSecretValue, ActionGetParameters, ActionDecrypt}, + Action: []string{actionGetSecretValue, actionGetParameters, actionDecrypt}, Resource: arns, }}, - }, nil + } } - return nil, nil + return nil } func uniqueStrings(items []string) []string { diff --git a/ecs/cloudformation_test.go b/ecs/cloudformation_test.go index 5f1d6ba4..02959120 100644 --- a/ecs/cloudformation_test.go +++ b/ecs/cloudformation_test.go @@ -45,7 +45,7 @@ func TestSimpleConvert(t *testing.T) { } func TestLogging(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: foo: image: hello_world @@ -64,7 +64,7 @@ x-aws-logs_retention: 10 } func TestEnvFile(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: foo: image: hello_world @@ -84,7 +84,7 @@ services: } func TestEnvFileAndEnv(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: foo: image: hello_world @@ -106,7 +106,7 @@ services: } func TestRollingUpdateLimits(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: foo: image: hello_world @@ -121,7 +121,7 @@ services: } func TestRollingUpdateExtension(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: foo: image: hello_world @@ -136,16 +136,17 @@ services: } func TestRolePolicy(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: foo: image: hello_world x-aws-pull_credentials: "secret" `) - role := template.Resources["FooTaskExecutionRole"].(*iam.Role) - assert.Check(t, role != nil) - assert.Check(t, role.ManagedPolicyArns[0] == ECSTaskExecutionPolicy) - assert.Check(t, role.ManagedPolicyArns[1] == ECRReadOnlyPolicy) + x := template.Resources["FooTaskExecutionRole"] + assert.Check(t, x != nil) + role := *(x.(*iam.Role)) + assert.Check(t, role.ManagedPolicyArns[0] == ecsTaskExecutionPolicy) + assert.Check(t, role.ManagedPolicyArns[1] == ecrReadOnlyPolicy) // We expect an extra policy has been created for x-aws-pull_credentials assert.Check(t, len(role.Policies) == 1) policy := role.Policies[0].PolicyDocument.(*PolicyDocument) @@ -155,7 +156,7 @@ services: } func TestMapNetworksToSecurityGroups(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: test: image: hello_world @@ -172,29 +173,31 @@ networks: assert.Check(t, template.Resources["TestPublicNetwork"] != nil) assert.Check(t, template.Resources["TestBacktierNetwork"] != nil) assert.Check(t, template.Resources["TestBacktierNetworkIngress"] != nil) - ingress := template.Resources["TestPublicNetworkIngress"].(*ec2.SecurityGroupIngress) - assert.Check(t, ingress != nil) + i := template.Resources["TestPublicNetworkIngress"] + assert.Check(t, i != nil) + ingress := *i.(*ec2.SecurityGroupIngress) assert.Check(t, ingress.SourceSecurityGroupId == cloudformation.Ref("TestPublicNetwork")) } func TestLoadBalancerTypeApplication(t *testing.T) { - template := convertYaml(t, "test123456789009876543211234567890", ` + template := convertYaml(t, ` services: test: image: nginx ports: - 80:80 `) - lb := template.Resources["TestLoadBalancer"].(*elasticloadbalancingv2.LoadBalancer) + lb := template.Resources["TestLoadBalancer"] assert.Check(t, lb != nil) - assert.Check(t, len(lb.Name) <= 32) - assert.Check(t, lb.Type == elbv2.LoadBalancerTypeEnumApplication) - assert.Check(t, len(lb.SecurityGroups) > 0) + loadBalancer := *lb.(*elasticloadbalancingv2.LoadBalancer) + assert.Check(t, len(loadBalancer.Name) <= 32) + assert.Check(t, loadBalancer.Type == elbv2.LoadBalancerTypeEnumApplication) + assert.Check(t, len(loadBalancer.SecurityGroups) > 0) } func TestNoLoadBalancerIfNoPortExposed(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: test: image: nginx @@ -209,20 +212,21 @@ services: } func TestServiceReplicas(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: test: image: nginx deploy: replicas: 10 `) - s := template.Resources["TestService"].(*ecs.Service) + s := template.Resources["TestService"] assert.Check(t, s != nil) - assert.Check(t, s.DesiredCount == 10) + service := *s.(*ecs.Service) + assert.Check(t, service.DesiredCount == 10) } func TestTaskSizeConvert(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: test: image: nginx @@ -239,7 +243,7 @@ services: assert.Equal(t, def.Cpu, "512") assert.Equal(t, def.Memory, "2048") - template = convertYaml(t, "test", ` + template = convertYaml(t, ` services: test: image: nginx @@ -257,7 +261,7 @@ services: assert.Equal(t, def.Memory, "8192") } func TestTaskSizeConvertFailure(t *testing.T) { - model := loadConfig(t, "test", ` + model := loadConfig(t, ` services: test: image: nginx @@ -273,7 +277,7 @@ services: } func TestLoadBalancerTypeNetwork(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: test: image: nginx @@ -281,13 +285,14 @@ services: - 80:80 - 88:88 `) - lb := template.Resources["TestLoadBalancer"].(*elasticloadbalancingv2.LoadBalancer) + lb := template.Resources["TestLoadBalancer"] assert.Check(t, lb != nil) - assert.Check(t, lb.Type == elbv2.LoadBalancerTypeEnumNetwork) + loadBalancer := *lb.(*elasticloadbalancingv2.LoadBalancer) + assert.Check(t, loadBalancer.Type == elbv2.LoadBalancerTypeEnumNetwork) } func TestServiceMapping(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: test: image: "image" @@ -326,7 +331,7 @@ func get(l []ecs.TaskDefinition_KeyValuePair, name string) string { } func TestResourcesHaveProjectTagSet(t *testing.T) { - template := convertYaml(t, "test", ` + template := convertYaml(t, ` services: test: image: nginx @@ -353,7 +358,7 @@ func convertResultAsString(t *testing.T, project *types.Project) string { backend := &ecsAPIService{} template, err := backend.convert(project) assert.NilError(t, err) - resultAsJSON, err := Marshall(template) + resultAsJSON, err := marshall(template) assert.NilError(t, err) return fmt.Sprintf("%s\n", string(resultAsJSON)) } @@ -368,15 +373,15 @@ func load(t *testing.T, paths ...string) *types.Project { return project } -func convertYaml(t *testing.T, name string, yaml string) *cloudformation.Template { - project := loadConfig(t, name, yaml) +func convertYaml(t *testing.T, yaml string) *cloudformation.Template { + project := loadConfig(t, yaml) backend := &ecsAPIService{} template, err := backend.convert(project) assert.NilError(t, err) return template } -func loadConfig(t *testing.T, name string, yaml string) *types.Project { +func loadConfig(t *testing.T, yaml string) *types.Project { dict, err := loader.ParseYAML([]byte(yaml)) assert.NilError(t, err) model, err := loader.Load(types.ConfigDetails{ diff --git a/ecs/colors.go b/ecs/colors.go index 22d609c8..1486e138 100644 --- a/ecs/colors.go +++ b/ecs/colors.go @@ -21,7 +21,7 @@ import ( "strconv" ) -var NAMES = []string{ +var names = []string{ "grey", "red", "green", @@ -32,14 +32,8 @@ var NAMES = []string{ "white", } -var COLORS map[string]ColorFunc - -// ColorFunc use ANSI codes to render colored text on console -type ColorFunc func(s string) string - -var Monochrome = func(s string) string { - return s -} +// colorFunc use ANSI codes to render colored text on console +type colorFunc func(s string) string func ansiColor(code, s string) string { return fmt.Sprintf("%s%s%s", ansi(code), s, ansi("0")) @@ -49,38 +43,38 @@ func ansi(code string) string { return fmt.Sprintf("\033[%sm", code) } -func makeColorFunc(code string) ColorFunc { +func makeColorFunc(code string) colorFunc { return func(s string) string { return ansiColor(code, s) } } -var Rainbow = make(chan ColorFunc) +var loop = make(chan colorFunc) func init() { - COLORS = map[string]ColorFunc{} - for i, name := range NAMES { - COLORS[name] = makeColorFunc(strconv.Itoa(30 + i)) - COLORS["intense_"+name] = makeColorFunc(strconv.Itoa(30+i) + ";1") + colors := map[string]colorFunc{} + for i, name := range names { + colors[name] = makeColorFunc(strconv.Itoa(30 + i)) + colors["intense_"+name] = makeColorFunc(strconv.Itoa(30+i) + ";1") } go func() { i := 0 - rainbow := []ColorFunc{ - COLORS["cyan"], - COLORS["yellow"], - COLORS["green"], - COLORS["magenta"], - COLORS["blue"], - COLORS["intense_cyan"], - COLORS["intense_yellow"], - COLORS["intense_green"], - COLORS["intense_magenta"], - COLORS["intense_blue"], + rainbow := []colorFunc{ + colors["cyan"], + colors["yellow"], + colors["green"], + colors["magenta"], + colors["blue"], + colors["intense_cyan"], + colors["intense_yellow"], + colors["intense_green"], + colors["intense_magenta"], + colors["intense_blue"], } for { - Rainbow <- rainbow[i] + loop <- rainbow[i] i = (i + 1) % len(rainbow) } }() diff --git a/ecs/compatibility.go b/ecs/compatibility.go index 316c6b88..803943d1 100644 --- a/ecs/compatibility.go +++ b/ecs/compatibility.go @@ -21,7 +21,7 @@ import ( "github.com/compose-spec/compose-go/types" ) -type FargateCompatibilityChecker struct { +type fargateCompatibilityChecker struct { compatibility.AllowList } @@ -68,13 +68,13 @@ var compatibleComposeAttributes = []string{ "secrets.file", } -func (c *FargateCompatibilityChecker) CheckImage(service *types.ServiceConfig) { +func (c *fargateCompatibilityChecker) CheckImage(service *types.ServiceConfig) { if service.Image == "" { c.Incompatible("service %s doesn't define a Docker image to run", service.Name) } } -func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortConfig) { +func (c *fargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortConfig) { if p.Published == 0 { p.Published = p.Target } @@ -83,7 +83,7 @@ func (c *FargateCompatibilityChecker) CheckPortsPublished(p *types.ServicePortCo } } -func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) { +func (c *fargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) { add := []string{} for _, cap := range service.CapAdd { switch cap { @@ -96,7 +96,7 @@ func (c *FargateCompatibilityChecker) CheckCapAdd(service *types.ServiceConfig) service.CapAdd = add } -func (c *FargateCompatibilityChecker) CheckLoggingDriver(config *types.LoggingConfig) { +func (c *fargateCompatibilityChecker) CheckLoggingDriver(config *types.LoggingConfig) { if config.Driver != "" && config.Driver != "awslogs" { c.Unsupported("services.logging.driver %s is not supported", config.Driver) } diff --git a/ecs/context.go b/ecs/context.go index e40202cb..c3b91023 100644 --- a/ecs/context.go +++ b/ecs/context.go @@ -103,7 +103,7 @@ func (h contextCreateAWSHelper) saveCredentials(profile string, accessKeyID stri p := credentials.SharedCredentialsProvider{Profile: profile} _, err := p.Retrieve() if err == nil { - return fmt.Errorf("credentials already exists!") + return fmt.Errorf("credentials already exists") } if err.(awserr.Error).Code() == "SharedCredsLoad" && err.(awserr.Error).Message() == "failed to load shared credentials file" { diff --git a/ecs/convert.go b/ecs/convert.go index 06557210..0f1f503b 100644 --- a/ecs/convert.go +++ b/ecs/convert.go @@ -44,10 +44,7 @@ func convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi if err != nil { return nil, err } - _, memReservation, err := toContainerReservation(service) - if err != nil { - return nil, err - } + _, memReservation := toContainerReservation(service) credential := getRepoCredentials(service) // override resolve.conf search directive to also search .local @@ -141,7 +138,11 @@ func convert(project *types.Project, service types.ServiceConfig) (*ecs.TaskDefi }, nil } -func createSecretsSideCar(project *types.Project, service types.ServiceConfig, logConfiguration *ecs.TaskDefinition_LogConfiguration) (ecs.TaskDefinition_Volume, ecs.TaskDefinition_MountPoint, ecs.TaskDefinition_ContainerDefinition, error) { +func createSecretsSideCar(project *types.Project, service types.ServiceConfig, logConfiguration *ecs.TaskDefinition_LogConfiguration) ( + ecs.TaskDefinition_Volume, + ecs.TaskDefinition_MountPoint, + ecs.TaskDefinition_ContainerDefinition, + error) { initContainerName := fmt.Sprintf("%s_Secrets_InitContainer", normalizeResourceName(service.Name)) secretsVolume := ecs.TaskDefinition_Volume{ Name: "secrets", @@ -166,7 +167,7 @@ func createSecretsSideCar(project *types.Project, service types.ServiceConfig, l ValueFrom: secretConfig.Name, }) var keys []string - if ext, ok := secretConfig.Extensions[ExtensionKeys]; ok { + if ext, ok := secretConfig.Extensions[extensionKeys]; ok { if key, ok := ext.(string); ok { keys = append(keys, key) } else { @@ -275,7 +276,7 @@ func toSystemControls(sysctls types.Mapping) []ecs.TaskDefinition_SystemControl return sys } -const MiB = 1024 * 1024 +const miB = 1024 * 1024 func toLimits(service types.ServiceConfig) (string, string, error) { // All possible cpu/mem values for Fargate @@ -315,9 +316,9 @@ func toLimits(service types.ServiceConfig) (string, string, error) { for _, cpu := range cpus { mem := cpuToMem[cpu] - if v <= cpu*MiB { + if v <= cpu*miB { for _, m := range mem { - if limits.MemoryBytes <= m*MiB { + if limits.MemoryBytes <= m*miB { cpuLimit = strconv.FormatInt(cpu, 10) memLimit = strconv.FormatInt(int64(m), 10) return cpuLimit, memLimit, nil @@ -328,19 +329,19 @@ func toLimits(service types.ServiceConfig) (string, string, error) { return "", "", fmt.Errorf("the resources requested are not supported by ECS/Fargate") } -func toContainerReservation(service types.ServiceConfig) (string, int, error) { +func toContainerReservation(service types.ServiceConfig) (string, int) { cpuReservation := ".0" memReservation := 0 if service.Deploy == nil { - return cpuReservation, memReservation, nil + return cpuReservation, memReservation } reservations := service.Deploy.Resources.Reservations if reservations == nil { - return cpuReservation, memReservation, nil + return cpuReservation, memReservation } - return reservations.NanoCPUs, int(reservations.MemoryBytes / MiB), nil + return reservations.NanoCPUs, int(reservations.MemoryBytes / miB) } func toPlacementConstraints(deploy *types.DeployConfig) []ecs.TaskDefinition_TaskDefinitionPlacementConstraint { @@ -467,7 +468,7 @@ func toHostEntryPtr(hosts types.HostsList) []ecs.TaskDefinition_HostEntry { func getRepoCredentials(service types.ServiceConfig) *ecs.TaskDefinition_RepositoryCredentials { // extract registry and namespace string from image name for key, value := range service.Extensions { - if key == ExtensionPullCredentials { + if key == extensionPullCredentials { return &ecs.TaskDefinition_RepositoryCredentials{CredentialsParameter: value.(string)} } } diff --git a/ecs/down.go b/ecs/down.go index 55e0dec9..4852f815 100644 --- a/ecs/down.go +++ b/ecs/down.go @@ -32,7 +32,7 @@ func (b *ecsAPIService) Down(ctx context.Context, options *cli.ProjectOptions) e if err != nil { return err } - return b.WaitStackCompletion(ctx, name, StackDelete) + return b.WaitStackCompletion(ctx, name, stackDelete) } func (b *ecsAPIService) projectName(options *cli.ProjectOptions) (string, error) { diff --git a/ecs/iam.go b/ecs/iam.go index 30670b0f..7a4510ea 100644 --- a/ecs/iam.go +++ b/ecs/iam.go @@ -17,12 +17,12 @@ package ecs const ( - ECSTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" - ECRReadOnlyPolicy = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ecsTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + ecrReadOnlyPolicy = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" - ActionGetSecretValue = "secretsmanager:GetSecretValue" - ActionGetParameters = "ssm:GetParameters" - ActionDecrypt = "kms:Decrypt" + actionGetSecretValue = "secretsmanager:GetSecretValue" + actionGetParameters = "ssm:GetParameters" + actionDecrypt = "kms:Decrypt" ) var assumeRolePolicyDocument = PolicyDocument{ @@ -38,12 +38,14 @@ var assumeRolePolicyDocument = PolicyDocument{ }, } +// PolicyDocument describes an IAM policy document // could alternatively depend on https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/master/cmd/clusterawsadm/api/iam/v1alpha1/types.go type PolicyDocument struct { Version string `json:",omitempty"` Statement []PolicyStatement `json:",omitempty"` } +// PolicyStatement describes an IAM policy statement type PolicyStatement struct { Effect string `json:",omitempty"` Action []string `json:",omitempty"` @@ -51,6 +53,7 @@ type PolicyStatement struct { Resource []string `json:",omitempty"` } +// PolicyPrincipal describes an IAM policy principal type PolicyPrincipal struct { Service string `json:",omitempty"` } diff --git a/ecs/list.go b/ecs/list.go index 02be946e..3c9ea5ad 100644 --- a/ecs/list.go +++ b/ecs/list.go @@ -35,7 +35,7 @@ func (b *ecsAPIService) Ps(ctx context.Context, options *cli.ProjectOptions) ([] if err != nil { return nil, err } - cluster := parameters[ParameterClusterName] + cluster := parameters[parameterClusterName] resources, err := b.SDK.ListStackResources(ctx, projectName) if err != nil { @@ -61,7 +61,7 @@ func (b *ecsAPIService) Ps(ctx context.Context, options *cli.ProjectOptions) ([] for i, state := range status { ports := []string{} - for _, lb := range state.LoadBalancers { + for _, lb := range state.Publishers { ports = append(ports, fmt.Sprintf( "%s:%d->%d/%s", lb.URL, diff --git a/ecs/logs.go b/ecs/logs.go index 58010fc1..84e913ea 100644 --- a/ecs/logs.go +++ b/ecs/logs.go @@ -40,7 +40,7 @@ func (b *ecsAPIService) Logs(ctx context.Context, options *cli.ProjectOptions, w } consumer := logConsumer{ - colors: map[string]ColorFunc{}, + colors: map[string]colorFunc{}, width: 0, writer: writer, } @@ -58,7 +58,7 @@ func (b *ecsAPIService) Logs(ctx context.Context, options *cli.ProjectOptions, w func (l *logConsumer) Log(service, container, message string) { cf, ok := l.colors[service] if !ok { - cf = <-Rainbow + cf = <-loop l.colors[service] = cf l.computeWidth() } @@ -81,7 +81,7 @@ func (l *logConsumer) computeWidth() { } type logConsumer struct { - colors map[string]ColorFunc + colors map[string]colorFunc width int writer io.Writer } diff --git a/ecs/marshall.go b/ecs/marshall.go index 8ef38a9d..d595e7c5 100644 --- a/ecs/marshall.go +++ b/ecs/marshall.go @@ -24,7 +24,7 @@ import ( "github.com/awslabs/goformation/v4/cloudformation" ) -func Marshall(template *cloudformation.Template) ([]byte, error) { +func marshall(template *cloudformation.Template) ([]byte, error) { raw, err := template.JSON() if err != nil { return nil, err diff --git a/ecs/sdk.go b/ecs/sdk.go index 3d499b1b..3f442976 100644 --- a/ecs/sdk.go +++ b/ecs/sdk.go @@ -22,11 +22,12 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/docker/api/compose" "github.com/docker/api/secrets" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" @@ -46,17 +47,16 @@ import ( ) type sdk struct { - sess *session.Session - ECS ecsiface.ECSAPI - EC2 ec2iface.EC2API - ELB elbv2iface.ELBV2API - CW cloudwatchlogsiface.CloudWatchLogsAPI - IAM iamiface.IAMAPI - CF cloudformationiface.CloudFormationAPI - SM secretsmanageriface.SecretsManagerAPI + ECS ecsiface.ECSAPI + EC2 ec2iface.EC2API + ELB elbv2iface.ELBV2API + CW cloudwatchlogsiface.CloudWatchLogsAPI + IAM iamiface.IAMAPI + CF cloudformationiface.CloudFormationAPI + SM secretsmanageriface.SecretsManagerAPI } -func NewSDK(sess *session.Session) sdk { +func newSDK(sess client.ConfigProvider) sdk { return sdk{ ECS: ecs.New(sess), EC2: ec2.New(sess), @@ -177,7 +177,7 @@ func (s sdk) StackExists(ctx context.Context, name string) (bool, error) { func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template, parameters map[string]string) error { logrus.Debug("Create CloudFormation stack") - json, err := Marshall(template) + json, err := marshall(template) if err != nil { return err } @@ -205,7 +205,7 @@ func (s sdk) CreateStack(ctx context.Context, name string, template *cf.Template func (s sdk) CreateChangeSet(ctx context.Context, name string, template *cf.Template, parameters map[string]string) (string, error) { logrus.Debug("Create CloudFormation Changeset") - json, err := Marshall(template) + json, err := marshall(template) if err != nil { return "", err } @@ -258,9 +258,9 @@ func (s sdk) UpdateStack(ctx context.Context, changeset string) error { } const ( - StackCreate = iota - StackUpdate - StackDelete + stackCreate = iota + stackUpdate + stackDelete ) func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) error { @@ -268,9 +268,9 @@ func (s sdk) WaitStackComplete(ctx context.Context, name string, operation int) StackName: aws.String(name), } switch operation { - case StackCreate: + case stackCreate: return s.CF.WaitUntilStackCreateCompleteWithContext(ctx, input) - case StackDelete: + case stackDelete: return s.CF.WaitUntilStackDeleteCompleteWithContext(ctx, input) default: return fmt.Errorf("internal error: unexpected stack operation %d", operation) @@ -322,14 +322,14 @@ func (s sdk) ListStackParameters(ctx context.Context, name string) (map[string]s return parameters, nil } -type StackResource struct { +type stackResource struct { LogicalID string Type string ARN string Status string } -func (s sdk) ListStackResources(ctx context.Context, name string) ([]StackResource, error) { +func (s sdk) ListStackResources(ctx context.Context, name string) ([]stackResource, error) { // FIXME handle pagination res, err := s.CF.ListStackResourcesWithContext(ctx, &cloudformation.ListStackResourcesInput{ StackName: aws.String(name), @@ -338,9 +338,9 @@ func (s sdk) ListStackResources(ctx context.Context, name string) ([]StackResour return nil, err } - resources := []StackResource{} + resources := []stackResource{} for _, r := range res.StackResourceSummaries { - resources = append(resources, StackResource{ + resources = append(resources, stackResource{ LogicalID: aws.StringValue(r.LogicalResourceId), Type: aws.StringValue(r.ResourceType), ARN: aws.StringValue(r.PhysicalResourceId), @@ -495,11 +495,11 @@ func (s sdk) DescribeServices(ctx context.Context, cluster string, arns []string return nil, err } status = append(status, compose.ServiceStatus{ - ID: aws.StringValue(service.ServiceName), - Name: name, - Replicas: int(aws.Int64Value(service.RunningCount)), - Desired: int(aws.Int64Value(service.DesiredCount)), - LoadBalancers: loadBalancers, + ID: aws.StringValue(service.ServiceName), + Name: name, + Replicas: int(aws.Int64Value(service.RunningCount)), + Desired: int(aws.Int64Value(service.DesiredCount)), + Publishers: loadBalancers, }) } return status, nil diff --git a/ecs/secrets/init.go b/ecs/secrets/init.go index 8ee28f29..729afe10 100644 --- a/ecs/secrets/init.go +++ b/ecs/secrets/init.go @@ -24,11 +24,13 @@ import ( "path/filepath" ) +// Secret define sensitive data to be bound as file type Secret struct { Name string Keys []string } +// CreateSecretFiles retrieve sensitive data from env and store as plain text a a file in path func CreateSecretFiles(secret Secret, path string) error { value, ok := os.LookupEnv(secret.Name) if !ok { diff --git a/ecs/secrets/main/main.go b/ecs/secrets/main/main.go index b70baf0d..d4b23876 100644 --- a/ecs/secrets/main/main.go +++ b/ecs/secrets/main/main.go @@ -28,21 +28,21 @@ const secretsFolder = "/run/secrets" func main() { if len(os.Args) != 2 { - fmt.Fprintf(os.Stderr, "usage: secrets ") + fmt.Fprint(os.Stderr, "usage: secrets ") os.Exit(1) } var input []secrets.Secret err := json.Unmarshal([]byte(os.Args[1]), &input) if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprint(os.Stderr, err.Error()) os.Exit(1) } for _, secret := range input { err := secrets.CreateSecretFiles(secret, secretsFolder) if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprint(os.Stderr, err.Error()) os.Exit(1) } } diff --git a/ecs/up.go b/ecs/up.go index a3e0b4dc..140f804f 100644 --- a/ecs/up.go +++ b/ecs/up.go @@ -67,20 +67,20 @@ func (b *ecsAPIService) Up(ctx context.Context, options *cli.ProjectOptions) err } parameters := map[string]string{ - ParameterClusterName: cluster, - ParameterVPCId: vpc, - ParameterSubnet1Id: subNets[0], - ParameterSubnet2Id: subNets[1], - ParameterLoadBalancerARN: lb, + parameterClusterName: cluster, + parameterVPCId: vpc, + parameterSubnet1Id: subNets[0], + parameterSubnet2Id: subNets[1], + parameterLoadBalancerARN: lb, } update, err := b.SDK.StackExists(ctx, project.Name) if err != nil { return err } - operation := StackCreate + operation := stackCreate if update { - operation = StackUpdate + operation = stackUpdate changeset, err := b.SDK.CreateChangeSet(ctx, project.Name, template, parameters) if err != nil { return err @@ -110,7 +110,7 @@ func (b *ecsAPIService) Up(ctx context.Context, options *cli.ProjectOptions) err func (b ecsAPIService) GetVPC(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if vpc, ok := project.Extensions[ExtensionVPC]; ok { + if vpc, ok := project.Extensions[extensionVPC]; ok { vpcID := vpc.(string) ok, err := b.SDK.VpcExists(ctx, vpcID) if err != nil { @@ -130,7 +130,7 @@ func (b ecsAPIService) GetVPC(ctx context.Context, project *types.Project) (stri func (b ecsAPIService) GetLoadBalancer(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if ext, ok := project.Extensions[ExtensionLB]; ok { + if ext, ok := project.Extensions[extensionLB]; ok { lb := ext.(string) ok, err := b.SDK.LoadBalancerExists(ctx, lb) if err != nil { @@ -146,7 +146,7 @@ func (b ecsAPIService) GetLoadBalancer(ctx context.Context, project *types.Proje func (b ecsAPIService) GetCluster(ctx context.Context, project *types.Project) (string, error) { //check compose file for custom VPC selected - if ext, ok := project.Extensions[ExtensionCluster]; ok { + if ext, ok := project.Extensions[extensionCluster]; ok { cluster := ext.(string) ok, err := b.SDK.ClusterExists(ctx, cluster) if err != nil { diff --git a/ecs/wait.go b/ecs/wait.go index 04632ee6..a6933dde 100644 --- a/ecs/wait.go +++ b/ecs/wait.go @@ -28,7 +28,7 @@ import ( "github.com/aws/aws-sdk-go/aws" ) -func (b *ecsAPIService) WaitStackCompletion(ctx context.Context, name string, operation int) error { +func (b *ecsAPIService) WaitStackCompletion(ctx context.Context, name string, operation int) error { //nolint:gocyclo knownEvents := map[string]struct{}{} // progress writer w := progress.ContextWriter(ctx) @@ -76,23 +76,23 @@ func (b *ecsAPIService) WaitStackCompletion(ctx context.Context, name string, op switch status { case "CREATE_COMPLETE": - if operation == StackCreate { + if operation == stackCreate { progressStatus = progress.Done } case "UPDATE_COMPLETE": - if operation == StackUpdate { + if operation == stackUpdate { progressStatus = progress.Done } case "DELETE_COMPLETE": - if operation == StackDelete { + if operation == stackDelete { progressStatus = progress.Done } default: if strings.HasSuffix(status, "_FAILED") { progressStatus = progress.Error if stackErr == nil { - operation = StackDelete + operation = stackDelete stackErr = fmt.Errorf(reason) } } diff --git a/ecs/x.go b/ecs/x.go index 1f2ba5ce..63af9c2e 100644 --- a/ecs/x.go +++ b/ecs/x.go @@ -17,15 +17,15 @@ package ecs const ( - ExtensionSecurityGroup = "x-aws-securitygroup" - ExtensionVPC = "x-aws-vpc" - ExtensionPullCredentials = "x-aws-pull_credentials" - ExtensionLB = "x-aws-loadbalancer" - ExtensionCluster = "x-aws-cluster" - ExtensionKeys = "x-aws-keys" - ExtensionMinPercent = "x-aws-min_percent" - ExtensionMaxPercent = "x-aws-max_percent" - ExtensionRetention = "x-aws-logs_retention" - ExtensionRole = "x-aws-role" - ExtensionManagedPolicies = "x-aws-policies" + extensionSecurityGroup = "x-aws-securitygroup" + extensionVPC = "x-aws-vpc" + extensionPullCredentials = "x-aws-pull_credentials" + extensionLB = "x-aws-loadbalancer" + extensionCluster = "x-aws-cluster" + extensionKeys = "x-aws-keys" + extensionMinPercent = "x-aws-min_percent" + extensionMaxPercent = "x-aws-max_percent" + extensionRetention = "x-aws-logs_retention" + extensionRole = "x-aws-role" + extensionManagedPolicies = "x-aws-policies" ) diff --git a/secrets/api.go b/secrets/api.go index f1231b7d..53fafe96 100644 --- a/secrets/api.go +++ b/secrets/api.go @@ -29,6 +29,7 @@ type Service interface { DeleteSecret(ctx context.Context, id string, recover bool) error } +// Secret hold sensitive data type Secret struct { ID string `json:"ID"` Name string `json:"Name"` @@ -38,6 +39,7 @@ type Secret struct { password string } +// NewSecret builds a secret func NewSecret(name, username, password, description string) Secret { return Secret{ Name: name, @@ -47,6 +49,7 @@ func NewSecret(name, username, password, description string) Secret { } } +// ToJSON marshall a Secret into JSON string func (s Secret) ToJSON() (string, error) { b, err := json.MarshalIndent(&s, "", "\t") if err != nil { @@ -55,6 +58,7 @@ func (s Secret) ToJSON() (string, error) { return string(b), nil } +// GetCredString marshall a Secret's sensitive data into JSON string func (s Secret) GetCredString() (string, error) { creds := map[string]string{ "username": s.username,