diff --git a/ecs/autoscaling.go b/ecs/autoscaling.go index 7a40938d..0c8cf251 100644 --- a/ecs/autoscaling.go +++ b/ecs/autoscaling.go @@ -62,7 +62,7 @@ func (b *ecsAPIService) createAutoscalingPolicy(project *types.Project, resource } // Why isn't this just the service ARN ????? - resourceID := cloudformation.Join("/", []string{"service", resources.cluster, cloudformation.GetAtt(serviceResourceName(service.Name), "Name")}) + resourceID := cloudformation.Join("/", []string{"service", resources.cluster.ID(), cloudformation.GetAtt(serviceResourceName(service.Name), "Name")}) target := fmt.Sprintf("%sScalableTarget", normalizeResourceName(service.Name)) template.Resources[target] = &applicationautoscaling.ScalableTarget{ diff --git a/ecs/aws.go b/ecs/aws.go index b5852017..915c1902 100644 --- a/ecs/aws.go +++ b/ecs/aws.go @@ -35,11 +35,11 @@ const ( // API hides aws-go-sdk into a simpler, focussed API subset type API interface { CheckRequirements(ctx context.Context, region string) error - ClusterExists(ctx context.Context, name string) (bool, error) + ResolveCluster(ctx context.Context, nameOrArn string) (awsResource, error) CreateCluster(ctx context.Context, name string) (string, error) CheckVPC(ctx context.Context, vpcID string) error GetDefaultVPC(ctx context.Context) (string, error) - GetSubNets(ctx context.Context, vpcID string) ([]string, error) + GetSubNets(ctx context.Context, vpcID string) ([]awsResource, error) GetRoleArn(ctx context.Context, name string) (string, error) StackExists(ctx context.Context, name string) (bool, error) CreateStack(ctx context.Context, name string, template []byte) error @@ -66,14 +66,14 @@ type API interface { getURLWithPortMapping(ctx context.Context, targetGroupArns []string) ([]compose.PortPublisher, error) ListTasks(ctx context.Context, cluster string, family string) ([]string, error) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string]string, error) - LoadBalancerType(ctx context.Context, arn string) (string, error) + ResolveLoadBalancer(ctx context.Context, nameOrArn string) (awsResource, string, error) GetLoadBalancerURL(ctx context.Context, arn string) (string, error) GetParameter(ctx context.Context, name string) (string, error) SecurityGroupExists(ctx context.Context, sg string) (bool, error) DeleteCapacityProvider(ctx context.Context, arn string) error DeleteAutoscalingGroup(ctx context.Context, arn string) error - FileSystemExists(ctx context.Context, id string) (bool, error) - FindFileSystem(ctx context.Context, tags map[string]string) (string, error) + ResolveFileSystem(ctx context.Context, id string) (awsResource, error) + FindFileSystem(ctx context.Context, tags map[string]string) (awsResource, error) CreateFileSystem(ctx context.Context, tags map[string]string) (string, error) DeleteFileSystem(ctx context.Context, id string) error } diff --git a/ecs/awsResources.go b/ecs/awsResources.go index 980329d8..27c1374d 100644 --- a/ecs/awsResources.go +++ b/ecs/awsResources.go @@ -38,13 +38,13 @@ import ( // awsResources hold the AWS component being used or created to support services definition type awsResources struct { - vpc string - subnets []string - cluster string - loadBalancer string + vpc string // shouldn't this also be an awsResource ? + subnets []awsResource + cluster awsResource + loadBalancer awsResource loadBalancerType string securityGroups map[string]string - filesystems map[string]string + filesystems map[string]awsResource } func (r *awsResources) serviceSecurityGroups(service types.ServiceConfig) []string { @@ -63,6 +63,63 @@ func (r *awsResources) allSecurityGroups() []string { return securityGroups } +func (r *awsResources) subnetsIDs() []string { + var ids []string + for _, r := range r.subnets { + ids = append(ids, r.ID()) + } + return ids +} + +// awsResource is abstract representation for any (existing or future) AWS resource that we can refer both by ID or full ARN +type awsResource interface { + ARN() string + ID() string +} + +// existingAWSResource hold references to an existing AWS component +type existingAWSResource struct { + arn string + id string +} + +func (r existingAWSResource) ARN() string { + return r.arn +} + +func (r existingAWSResource) ID() string { + return r.id +} + +// cloudformationResource hold references to a future AWS resource managed by CloudFormation +// to be used by CloudFormation resources where Ref returns the Amazon Resource ID +type cloudformationResource struct { + logicalName string +} + +func (r cloudformationResource) ARN() string { + return cloudformation.GetAtt(r.logicalName, "Arn") +} + +func (r cloudformationResource) ID() string { + return cloudformation.Ref(r.logicalName) +} + +// cloudformationARNResource hold references to a future AWS resource managed by CloudFormation +// to be used by CloudFormation resources where Ref returns the Amazon Resource Name (ARN) +type cloudformationARNResource struct { + logicalName string + nameProperty string +} + +func (r cloudformationARNResource) ARN() string { + return cloudformation.Ref(r.logicalName) +} + +func (r cloudformationARNResource) ID() string { + return cloudformation.GetAtt(r.logicalName, r.nameProperty) +} + // parse look into compose project for configured resource to use, and check they are valid func (b *ecsAPIService) parse(ctx context.Context, project *types.Project, template *cloudformation.Template) (awsResources, error) { r := awsResources{} @@ -90,28 +147,28 @@ func (b *ecsAPIService) parse(ctx context.Context, project *types.Project, templ return r, nil } -func (b *ecsAPIService) parseClusterExtension(ctx context.Context, project *types.Project, template *cloudformation.Template) (string, error) { +func (b *ecsAPIService) parseClusterExtension(ctx context.Context, project *types.Project, template *cloudformation.Template) (awsResource, error) { if x, ok := project.Extensions[extensionCluster]; ok { - cluster := x.(string) - ok, err := b.aws.ClusterExists(ctx, cluster) + nameOrArn := x.(string) // can be name _or_ ARN. + cluster, err := b.aws.ResolveCluster(ctx, nameOrArn) if err != nil { - return "", err + return nil, err } if !ok { - return "", errors.Wrapf(errdefs.ErrNotFound, "cluster %q does not exist", cluster) + return nil, errors.Wrapf(errdefs.ErrNotFound, "cluster %q does not exist", cluster) } - template.Metadata["Cluster"] = cluster + template.Metadata["Cluster"] = cluster.ARN() return cluster, nil } - return "", nil + return nil, nil } -func (b *ecsAPIService) parseVPCExtension(ctx context.Context, project *types.Project) (string, []string, error) { +func (b *ecsAPIService) parseVPCExtension(ctx context.Context, project *types.Project) (string, []awsResource, error) { var vpc string if x, ok := project.Extensions[extensionVPC]; ok { - vpc = x.(string) - err := b.aws.CheckVPC(ctx, vpc) + vpcID := x.(string) + err := b.aws.CheckVPC(ctx, vpcID) if err != nil { return "", nil, err } @@ -134,22 +191,22 @@ func (b *ecsAPIService) parseVPCExtension(ctx context.Context, project *types.Pr return vpc, subNets, nil } -func (b *ecsAPIService) parseLoadBalancerExtension(ctx context.Context, project *types.Project) (string, string, error) { +func (b *ecsAPIService) parseLoadBalancerExtension(ctx context.Context, project *types.Project) (awsResource, string, error) { if x, ok := project.Extensions[extensionLoadBalancer]; ok { - loadBalancer := x.(string) - loadBalancerType, err := b.aws.LoadBalancerType(ctx, loadBalancer) + nameOrArn := x.(string) + loadBalancer, loadBalancerType, err := b.aws.ResolveLoadBalancer(ctx, nameOrArn) if err != nil { - return "", "", err + return nil, "", err } required := getRequiredLoadBalancerType(project) if loadBalancerType != required { - return "", "", fmt.Errorf("load balancer %s is of type %s, project require a %s", loadBalancer, loadBalancerType, required) + return nil, "", fmt.Errorf("load balancer %q is of type %s, project require a %s", nameOrArn, loadBalancerType, required) } - return loadBalancer, loadBalancerType, nil + return loadBalancer, loadBalancerType, err } - return "", "", nil + return nil, "", nil } func (b *ecsAPIService) parseExternalNetworks(ctx context.Context, project *types.Project) (map[string]string, error) { @@ -179,18 +236,15 @@ func (b *ecsAPIService) parseExternalNetworks(ctx context.Context, project *type return securityGroups, nil } -func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types.Project) (map[string]string, error) { - filesystems := make(map[string]string, len(project.Volumes)) +func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types.Project) (map[string]awsResource, error) { + filesystems := make(map[string]awsResource, len(project.Volumes)) for name, vol := range project.Volumes { if vol.External.External { - exists, err := b.aws.FileSystemExists(ctx, vol.Name) + arn, err := b.aws.ResolveFileSystem(ctx, vol.Name) if err != nil { return nil, err } - if !exists { - return nil, errors.Wrapf(errdefs.ErrNotFound, "EFS file system %q doesn't exist", vol.Name) - } - filesystems[name] = vol.Name + filesystems[name] = arn continue } @@ -199,12 +253,12 @@ func (b *ecsAPIService) parseExternalVolumes(ctx context.Context, project *types compose.ProjectTag: project.Name, compose.VolumeTag: name, } - id, err := b.aws.FindFileSystem(ctx, tags) + fileSystem, err := b.aws.FindFileSystem(ctx, tags) if err != nil { return nil, err } - if id != "" { - filesystems[name] = id + if fileSystem != nil { + filesystems[name] = fileSystem } } return filesystems, nil @@ -223,14 +277,14 @@ func (b *ecsAPIService) ensureResources(resources *awsResources, project *types. } func (b *ecsAPIService) ensureCluster(r *awsResources, project *types.Project, template *cloudformation.Template) { - if r.cluster != "" { + if r.cluster != nil { return } template.Resources["Cluster"] = &ecs.Cluster{ ClusterName: project.Name, Tags: projectTags(project), } - r.cluster = cloudformation.Ref("Cluster") + r.cluster = cloudformationResource{logicalName: "Cluster"} } func (b *ecsAPIService) ensureNetworks(r *awsResources, project *types.Project, template *cloudformation.Template) { @@ -319,13 +373,13 @@ func (b *ecsAPIService) ensureVolumes(r *awsResources, project *types.Project, t ThroughputMode: throughputMode, AWSCloudFormationDeletionPolicy: "Retain", } - r.filesystems[name] = cloudformation.Ref(n) + r.filesystems[name] = cloudformationResource{logicalName: n} } return nil } func (b *ecsAPIService) ensureLoadBalancer(r *awsResources, project *types.Project, template *cloudformation.Template) { - if r.loadBalancer != "" { + if r.loadBalancer != nil { return } if allServices(project.Services, func(it types.ServiceConfig) bool { @@ -345,11 +399,14 @@ func (b *ecsAPIService) ensureLoadBalancer(r *awsResources, project *types.Proje template.Resources["LoadBalancer"] = &elasticloadbalancingv2.LoadBalancer{ Scheme: elbv2.LoadBalancerSchemeEnumInternetFacing, SecurityGroups: securityGroups, - Subnets: r.subnets, + Subnets: r.subnetsIDs(), Tags: projectTags(project), Type: balancerType, } - r.loadBalancer = cloudformation.Ref("LoadBalancer") + r.loadBalancer = cloudformationARNResource{ + logicalName: "LoadBalancer", + nameProperty: "LoadBalancerName", + } r.loadBalancerType = balancerType } diff --git a/ecs/aws_mock.go b/ecs/aws_mock.go index 504de0c5..5c669cb5 100644 --- a/ecs/aws_mock.go +++ b/ecs/aws_mock.go @@ -6,12 +6,13 @@ package ecs import ( context "context" + reflect "reflect" + cloudformation "github.com/aws/aws-sdk-go/service/cloudformation" ecs "github.com/aws/aws-sdk-go/service/ecs" compose "github.com/docker/compose-cli/api/compose" secrets "github.com/docker/compose-cli/api/secrets" gomock "github.com/golang/mock/gomock" - reflect "reflect" ) // MockAPI is a mock of API interface @@ -65,21 +66,6 @@ func (mr *MockAPIMockRecorder) CheckVPC(arg0, arg1 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckVPC", reflect.TypeOf((*MockAPI)(nil).CheckVPC), arg0, arg1) } -// ClusterExists mocks base method -func (m *MockAPI) ClusterExists(arg0 context.Context, arg1 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ClusterExists", arg0, arg1) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ClusterExists indicates an expected call of ClusterExists -func (mr *MockAPIMockRecorder) ClusterExists(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterExists", reflect.TypeOf((*MockAPI)(nil).ClusterExists), arg0, arg1) -} - // CreateChangeSet mocks base method func (m *MockAPI) CreateChangeSet(arg0 context.Context, arg1 string, arg2 []byte) (string, error) { m.ctrl.T.Helper() @@ -254,26 +240,11 @@ func (mr *MockAPIMockRecorder) DescribeStackEvents(arg0, arg1 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeStackEvents", reflect.TypeOf((*MockAPI)(nil).DescribeStackEvents), arg0, arg1) } -// FileSystemExists mocks base method -func (m *MockAPI) FileSystemExists(arg0 context.Context, arg1 string) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FileSystemExists", arg0, arg1) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// FileSystemExists indicates an expected call of FileSystemExists -func (mr *MockAPIMockRecorder) FileSystemExists(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileSystemExists", reflect.TypeOf((*MockAPI)(nil).FileSystemExists), arg0, arg1) -} - // FindFileSystem mocks base method -func (m *MockAPI) FindFileSystem(arg0 context.Context, arg1 map[string]string) (string, error) { +func (m *MockAPI) FindFileSystem(arg0 context.Context, arg1 map[string]string) (awsResource, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FindFileSystem", arg0, arg1) - ret0, _ := ret[0].(string) + ret0, _ := ret[0].(awsResource) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -439,10 +410,10 @@ func (mr *MockAPIMockRecorder) GetStackID(arg0, arg1 interface{}) *gomock.Call { } // GetSubNets mocks base method -func (m *MockAPI) GetSubNets(arg0 context.Context, arg1 string) ([]string, error) { +func (m *MockAPI) GetSubNets(arg0 context.Context, arg1 string) ([]awsResource, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSubNets", arg0, arg1) - ret0, _ := ret[0].([]string) + ret0, _ := ret[0].([]awsResource) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -573,19 +544,50 @@ func (mr *MockAPIMockRecorder) ListTasks(arg0, arg1, arg2 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTasks", reflect.TypeOf((*MockAPI)(nil).ListTasks), arg0, arg1, arg2) } -// LoadBalancerType mocks base method -func (m *MockAPI) LoadBalancerType(arg0 context.Context, arg1 string) (string, error) { +// ResolveCluster mocks base method +func (m *MockAPI) ResolveCluster(arg0 context.Context, arg1 string) (awsResource, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadBalancerType", arg0, arg1) - ret0, _ := ret[0].(string) + ret := m.ctrl.Call(m, "ResolveCluster", arg0, arg1) + ret0, _ := ret[0].(awsResource) ret1, _ := ret[1].(error) return ret0, ret1 } -// LoadBalancerType indicates an expected call of LoadBalancerType -func (mr *MockAPIMockRecorder) LoadBalancerType(arg0, arg1 interface{}) *gomock.Call { +// ResolveCluster indicates an expected call of ResolveCluster +func (mr *MockAPIMockRecorder) ResolveCluster(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerType", reflect.TypeOf((*MockAPI)(nil).LoadBalancerType), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveCluster", reflect.TypeOf((*MockAPI)(nil).ResolveCluster), arg0, arg1) +} + +// ResolveFileSystem mocks base method +func (m *MockAPI) ResolveFileSystem(arg0 context.Context, arg1 string) (awsResource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveFileSystem", arg0, arg1) + ret0, _ := ret[0].(awsResource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveFileSystem indicates an expected call of ResolveFileSystem +func (mr *MockAPIMockRecorder) ResolveFileSystem(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveFileSystem", reflect.TypeOf((*MockAPI)(nil).ResolveFileSystem), arg0, arg1) +} + +// ResolveLoadBalancer mocks base method +func (m *MockAPI) ResolveLoadBalancer(arg0 context.Context, arg1 string) (awsResource, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveLoadBalancer", arg0, arg1) + ret0, _ := ret[0].(awsResource) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ResolveLoadBalancer indicates an expected call of ResolveLoadBalancer +func (mr *MockAPIMockRecorder) ResolveLoadBalancer(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveLoadBalancer", reflect.TypeOf((*MockAPI)(nil).ResolveLoadBalancer), arg0, arg1) } // SecurityGroupExists mocks base method diff --git a/ecs/backend.go b/ecs/backend.go index b7218191..7e499437 100644 --- a/ecs/backend.go +++ b/ecs/backend.go @@ -19,9 +19,6 @@ package ecs import ( "context" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/docker/compose-cli/api/compose" "github.com/docker/compose-cli/api/containers" "github.com/docker/compose-cli/api/resources" @@ -32,6 +29,9 @@ import ( "github.com/docker/compose-cli/context/cloud" "github.com/docker/compose-cli/context/store" "github.com/docker/compose-cli/errdefs" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" ) const backendType = store.EcsContextType diff --git a/ecs/cloudformation.go b/ecs/cloudformation.go index e284dd51..58dbaaea 100644 --- a/ecs/cloudformation.go +++ b/ecs/cloudformation.go @@ -77,6 +77,8 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c b.createNFSMountTarget(project, resources, template) + b.createAccessPoints(project, resources, template) + for _, service := range project.Services { err := b.createService(project, service, template, resources) if err != nil { @@ -96,7 +98,7 @@ func (b *ecsAPIService) convert(ctx context.Context, project *types.Project) (*c func (b *ecsAPIService) createService(project *types.Project, service types.ServiceConfig, template *cloudformation.Template, resources awsResources) error { taskExecutionRole := b.createTaskExecutionRole(project, service, template) - taskRole := b.createTaskRole(project, service, template) + taskRole := b.createTaskRole(project, service, template, resources) definition, err := b.createTaskDefinition(project, service, resources) if err != nil { @@ -166,7 +168,7 @@ func (b *ecsAPIService) createService(project *types.Project, service types.Serv template.Resources[serviceResourceName(service.Name)] = &ecs.Service{ AWSCloudFormationDependsOn: dependsOn, - Cluster: resources.cluster, + Cluster: resources.cluster.ARN(), DesiredCount: desiredCount, DeploymentController: &ecs.Service_DeploymentController{ Type: ecsapi.DeploymentControllerTypeEcs, @@ -182,7 +184,7 @@ func (b *ecsAPIService) createService(project *types.Project, service types.Serv AwsvpcConfiguration: &ecs.Service_AwsVpcConfiguration{ AssignPublicIp: assignPublicIP, SecurityGroups: resources.serviceSecurityGroups(service), - Subnets: resources.subnets, + Subnets: resources.subnetsIDs(), }, }, PlatformVersion: platformVersion, @@ -287,7 +289,7 @@ func computeRollingUpdateLimits(service types.ServiceConfig) (int, int, error) { func (b *ecsAPIService) createListener(service types.ServiceConfig, port types.ServicePortConfig, template *cloudformation.Template, - targetGroupName string, loadBalancerARN string, protocol string) string { + targetGroupName string, loadBalancer awsResource, protocol string) string { listenerName := fmt.Sprintf( "%s%s%dListener", normalizeResourceName(service.Name), @@ -309,7 +311,7 @@ func (b *ecsAPIService) createListener(service types.ServiceConfig, port types.S Type: elbv2.ActionTypeEnumForward, }, }, - LoadBalancerArn: loadBalancerARN, + LoadBalancerArn: loadBalancer.ARN(), Protocol: protocol, Port: int(port.Target), } @@ -375,14 +377,21 @@ func (b *ecsAPIService) createTaskExecutionRole(project *types.Project, service return taskExecutionRole } -func (b *ecsAPIService) createTaskRole(project *types.Project, service types.ServiceConfig, template *cloudformation.Template) string { +func (b *ecsAPIService) createTaskRole(project *types.Project, service types.ServiceConfig, template *cloudformation.Template, resources awsResources) string { taskRole := fmt.Sprintf("%sTaskRole", normalizeResourceName(service.Name)) rolePolicies := []iam.Role_Policy{} if roles, ok := service.Extensions[extensionRole]; ok { rolePolicies = append(rolePolicies, iam.Role_Policy{ + PolicyName: fmt.Sprintf("%s%sPolicy", normalizeResourceName(project.Name), normalizeResourceName(service.Name)), PolicyDocument: roles, }) } + for _, vol := range service.Volumes { + rolePolicies = append(rolePolicies, iam.Role_Policy{ + PolicyName: fmt.Sprintf("%s%sVolumeMountPolicy", normalizeResourceName(project.Name), normalizeResourceName(service.Name)), + PolicyDocument: volumeMountPolicyDocument(vol.Source, resources.filesystems[vol.Source].ARN()), + }) + } managedPolicies := []string{} if v, ok := service.Extensions[extensionManagedPolicies]; ok { for _, s := range v.([]interface{}) { diff --git a/ecs/cloudformation_test.go b/ecs/cloudformation_test.go index 06881df0..be5d68df 100644 --- a/ecs/cloudformation_test.go +++ b/ecs/cloudformation_test.go @@ -367,7 +367,7 @@ volumes: external: true name: fs-123abc `, useDefaultVPC, func(m *MockAPIMockRecorder) { - m.FileSystemExists(gomock.Any(), "fs-123abc").Return(true, nil) + m.ResolveFileSystem(gomock.Any(), "fs-123abc").Return(existingAWSResource{id: "fs-123abc"}, nil) }) s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget) assert.Check(t, s != nil) @@ -395,7 +395,7 @@ volumes: m.FindFileSystem(gomock.Any(), map[string]string{ compose.ProjectTag: t.Name(), compose.VolumeTag: "db-data", - }).Return("", nil) + }).Return(nil, nil) }) n := volumeResourceName("db-data") f := template.Resources[n].(*efs.FileSystem) @@ -411,6 +411,25 @@ volumes: assert.Equal(t, s.FileSystemId, cloudformation.Ref(n)) //nolint:staticcheck } +func TestCreateAccessPoint(t *testing.T) { + template := convertYaml(t, ` +services: + test: + image: nginx +volumes: + db-data: + driver_opts: + uid: 1002 + gid: 1002 +`, useDefaultVPC, func(m *MockAPIMockRecorder) { + m.FindFileSystem(gomock.Any(), gomock.Any()).Return(nil, nil) + }) + a := template.Resources["DbdataAccessPoint"].(*efs.AccessPoint) + assert.Check(t, a != nil) + assert.Equal(t, a.PosixUser.Uid, "1002") //nolint:staticcheck + assert.Equal(t, a.PosixUser.Gid, "1002") //nolint:staticcheck +} + func TestReusePreviousVolume(t *testing.T) { template := convertYaml(t, ` services: @@ -422,7 +441,7 @@ volumes: m.FindFileSystem(gomock.Any(), map[string]string{ compose.ProjectTag: t.Name(), compose.VolumeTag: "db-data", - }).Return("fs-123abc", nil) + }).Return(existingAWSResource{id: "fs-123abc"}, nil) }) s := template.Resources["DbdataNFSMountTargetOnSubnet1"].(*efs.MountTarget) assert.Check(t, s != nil) @@ -499,7 +518,10 @@ services: test: image: nginx `, useDefaultVPC, func(m *MockAPIMockRecorder) { - m.ClusterExists(gomock.Any(), "arn:aws:ecs:region:account:cluster/name").Return(true, nil) + m.ResolveCluster(gomock.Any(), "arn:aws:ecs:region:account:cluster/name").Return(existingAWSResource{ + arn: "arn:aws:ecs:region:account:cluster/name", + id: "name", + }, nil) }) assert.Equal(t, template.Metadata["Cluster"], "arn:aws:ecs:region:account:cluster/name") } @@ -548,7 +570,10 @@ func getMainContainer(def *ecs.TaskDefinition, t *testing.T) ecs.TaskDefinition_ func useDefaultVPC(m *MockAPIMockRecorder) { m.GetDefaultVPC(gomock.Any()).Return("vpc-123", nil) - m.GetSubNets(gomock.Any(), "vpc-123").Return([]string{"subnet1", "subnet2"}, nil) + m.GetSubNets(gomock.Any(), "vpc-123").Return([]awsResource{ + existingAWSResource{id: "subnet1"}, + existingAWSResource{id: "subnet2"}, + }, nil) } func useGPU(m *MockAPIMockRecorder) { diff --git a/ecs/convert.go b/ecs/convert.go index 436e0bb3..aef6007f 100644 --- a/ecs/convert.go +++ b/ecs/convert.go @@ -81,11 +81,15 @@ func (b *ecsAPIService) createTaskDefinition(project *types.Project, service typ } for _, v := range service.Volumes { - volume := project.Volumes[v.Source] + n := fmt.Sprintf("%sAccessPoint", normalizeResourceName(v.Source)) volumes = append(volumes, ecs.TaskDefinition_Volume{ EFSVolumeConfiguration: &ecs.TaskDefinition_EFSVolumeConfiguration{ - FilesystemId: resources.filesystems[v.Source], - RootDirectory: volume.DriverOpts["root_directory"], + AuthorizationConfig: &ecs.TaskDefinition_AuthorizationConfig{ + AccessPointId: cloudformation.Ref(n), + IAM: "ENABLED", + }, + FilesystemId: resources.filesystems[v.Source].ID(), + TransitEncryption: "ENABLED", }, Name: v.Source, }) diff --git a/ecs/ec2.go b/ecs/ec2.go index fae76ed5..f7899eb8 100644 --- a/ecs/ec2.go +++ b/ecs/ec2.go @@ -65,7 +65,7 @@ func (b *ecsAPIService) createCapacityProvider(ctx context.Context, project *typ LaunchConfigurationName: cloudformation.Ref("LaunchConfiguration"), MaxSize: "10", //TODO MinSize: "1", - VPCZoneIdentifier: resources.subnets, + VPCZoneIdentifier: resources.subnetsIDs(), } userData := base64.StdEncoding.EncodeToString([]byte( diff --git a/ecs/iam.go b/ecs/iam.go index 5b061172..f6b80ed2 100644 --- a/ecs/iam.go +++ b/ecs/iam.go @@ -16,6 +16,12 @@ package ecs +import ( + "fmt" + + "github.com/awslabs/goformation/v4/cloudformation" +) + const ( ecsTaskExecutionPolicy = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" ecrReadOnlyPolicy = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" @@ -49,7 +55,31 @@ func policyDocument(service string) PolicyDocument { }, }, } +} +func volumeMountPolicyDocument(volume string, filesystem string) PolicyDocument { + ap := fmt.Sprintf("%sAccessPoint", normalizeResourceName(volume)) + return PolicyDocument{ + Version: "2012-10-17", // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html + Statement: []PolicyStatement{ + { + Effect: "Allow", + Resource: []string{ + filesystem, + }, + Action: []string{ + "elasticfilesystem:ClientMount", + "elasticfilesystem:ClientWrite", + "elasticfilesystem:ClientRootAccess", + }, + Condition: Condition{ + StringEquals: map[string]string{ + "elasticfilesystem:AccessPointArn": cloudformation.Ref(ap), + }, + }, + }, + }, + } } // PolicyDocument describes an IAM policy document @@ -65,9 +95,16 @@ type PolicyStatement struct { Action []string `json:",omitempty"` Principal PolicyPrincipal `json:",omitempty"` Resource []string `json:",omitempty"` + Condition Condition `json:",omitempty"` } // PolicyPrincipal describes an IAM policy principal type PolicyPrincipal struct { Service string `json:",omitempty"` } + +// Condition is the map of all conditions in the statement entry. +type Condition struct { + StringEquals map[string]string `json:",omitempty"` + Bool map[string]string `json:",omitempty"` +} diff --git a/ecs/sdk.go b/ecs/sdk.go index 77ee60d4..08b1e6d7 100644 --- a/ecs/sdk.go +++ b/ecs/sdk.go @@ -29,6 +29,7 @@ import ( "github.com/docker/compose-cli/internal" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/autoscaling" @@ -107,15 +108,22 @@ func (s sdk) CheckRequirements(ctx context.Context, region string) error { return nil } -func (s sdk) ClusterExists(ctx context.Context, name string) (bool, error) { - logrus.Debug("CheckRequirements if cluster was already created: ", name) +func (s sdk) ResolveCluster(ctx context.Context, nameOrArn string) (awsResource, error) { + logrus.Debug("CheckRequirements if cluster was already created: ", nameOrArn) clusters, err := s.ECS.DescribeClustersWithContext(ctx, &ecs.DescribeClustersInput{ - Clusters: []*string{aws.String(name)}, + Clusters: []*string{aws.String(nameOrArn)}, }) if err != nil { - return false, err + return nil, err } - return len(clusters.Clusters) > 0, nil + if len(clusters.Clusters) == 0 { + return nil, errors.Wrapf(errdefs.ErrNotFound, "cluster %q does not exist", nameOrArn) + } + it := clusters.Clusters[0] + return existingAWSResource{ + arn: aws.StringValue(it.ClusterArn), + id: aws.StringValue(it.ClusterName), + }, nil } func (s sdk) CreateCluster(ctx context.Context, name string) (string, error) { @@ -139,7 +147,7 @@ func (s sdk) CheckVPC(ctx context.Context, vpcID string) error { if !*output.EnableDnsSupport.Value { return fmt.Errorf("VPC %q doesn't have DNS resolution enabled", vpcID) } - return err + return nil } func (s sdk) GetDefaultVPC(ctx context.Context) (string, error) { @@ -161,7 +169,7 @@ func (s sdk) GetDefaultVPC(ctx context.Context) (string, error) { return *vpcs.Vpcs[0].VpcId, nil } -func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) { +func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]awsResource, error) { logrus.Debug("Retrieve SubNets") subnets, err := s.EC2.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{ DryRun: nil, @@ -176,9 +184,12 @@ func (s sdk) GetSubNets(ctx context.Context, vpcID string) ([]string, error) { return nil, err } - ids := []string{} + ids := []awsResource{} for _, subnet := range subnets.Subnets { - ids = append(ids, *subnet.SubnetId) + ids = append(ids, existingAWSResource{ + arn: aws.StringValue(subnet.SubnetArn), + id: aws.StringValue(subnet.SubnetId), + }) } return ids, nil } @@ -785,18 +796,31 @@ func (s sdk) GetPublicIPs(ctx context.Context, interfaces ...string) (map[string return publicIPs, nil } -func (s sdk) LoadBalancerType(ctx context.Context, arn string) (string, error) { - logrus.Debug("Check if LoadBalancer exists: ", arn) +func (s sdk) ResolveLoadBalancer(ctx context.Context, nameOrarn string) (awsResource, string, error) { + logrus.Debug("Check if LoadBalancer exists: ", nameOrarn) + var arns []*string + var names []*string + if arn.IsARN(nameOrarn) { + arns = append(arns, aws.String(nameOrarn)) + } else { + names = append(names, aws.String(nameOrarn)) + } + lbs, err := s.ELB.DescribeLoadBalancersWithContext(ctx, &elbv2.DescribeLoadBalancersInput{ - LoadBalancerArns: []*string{aws.String(arn)}, + LoadBalancerArns: arns, + Names: names, }) if err != nil { - return "", err + return nil, "", err } if len(lbs.LoadBalancers) == 0 { - return "", fmt.Errorf("load balancer does not exist: %s", arn) + return nil, "", errors.Wrapf(errdefs.ErrNotFound, "load balancer %q does not exist", nameOrarn) } - return aws.StringValue(lbs.LoadBalancers[0].Type), nil + it := lbs.LoadBalancers[0] + return existingAWSResource{ + arn: aws.StringValue(it.LoadBalancerArn), + id: aws.StringValue(it.LoadBalancerName), + }, aws.StringValue(it.Type), nil } func (s sdk) GetLoadBalancerURL(ctx context.Context, arn string) (string, error) { @@ -864,32 +888,42 @@ func (s sdk) DeleteAutoscalingGroup(ctx context.Context, arn string) error { return err } -func (s sdk) FileSystemExists(ctx context.Context, id string) (bool, error) { +func (s sdk) ResolveFileSystem(ctx context.Context, id string) (awsResource, error) { desc, err := s.EFS.DescribeFileSystemsWithContext(ctx, &efs.DescribeFileSystemsInput{ FileSystemId: aws.String(id), }) if err != nil { - return false, err + return nil, err } - return len(desc.FileSystems) > 0, nil + if len(desc.FileSystems) == 0 { + return nil, errors.Wrapf(errdefs.ErrNotFound, "EFS file system %q doesn't exist", id) + } + it := desc.FileSystems[0] + return existingAWSResource{ + arn: aws.StringValue(it.FileSystemArn), + id: aws.StringValue(it.FileSystemId), + }, nil } -func (s sdk) FindFileSystem(ctx context.Context, tags map[string]string) (string, error) { +func (s sdk) FindFileSystem(ctx context.Context, tags map[string]string) (awsResource, error) { var token *string for { desc, err := s.EFS.DescribeFileSystemsWithContext(ctx, &efs.DescribeFileSystemsInput{ Marker: token, }) if err != nil { - return "", err + return nil, err } for _, filesystem := range desc.FileSystems { if containsAll(filesystem.Tags, tags) { - return aws.StringValue(filesystem.FileSystemId), nil + return existingAWSResource{ + arn: aws.StringValue(filesystem.FileSystemArn), + id: aws.StringValue(filesystem.FileSystemId), + }, nil } } if desc.NextMarker == token { - return "", nil + return nil, nil } token = desc.NextMarker } diff --git a/ecs/testdata/simple/simple-cloudformation-conversion.golden b/ecs/testdata/simple/simple-cloudformation-conversion.golden index 7e29555a..51812113 100644 --- a/ecs/testdata/simple/simple-cloudformation-conversion.golden +++ b/ecs/testdata/simple/simple-cloudformation-conversion.golden @@ -98,7 +98,10 @@ ], "Properties": { "Cluster": { - "Ref": "Cluster" + "Fn::GetAtt": [ + "Cluster", + "Arn" + ] }, "DeploymentConfiguration": { "MaximumPercent": 200, @@ -299,6 +302,7 @@ "Action": [ "sts:AssumeRole" ], + "Condition": {}, "Effect": "Allow", "Principal": { "Service": "ecs-tasks.amazonaws.com" diff --git a/ecs/volumes.go b/ecs/volumes.go index a5d4ea3a..939ea9f7 100644 --- a/ecs/volumes.go +++ b/ecs/volumes.go @@ -19,6 +19,8 @@ package ecs import ( "fmt" + "github.com/docker/compose-cli/api/compose" + "github.com/awslabs/goformation/v4/cloudformation" "github.com/awslabs/goformation/v4/cloudformation/efs" "github.com/compose-spec/compose-go/types" @@ -27,11 +29,11 @@ import ( func (b *ecsAPIService) createNFSMountTarget(project *types.Project, resources awsResources, template *cloudformation.Template) { for volume := range project.Volumes { for _, subnet := range resources.subnets { - name := fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet)) + name := fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet.ID())) template.Resources[name] = &efs.MountTarget{ - FileSystemId: resources.filesystems[volume], + FileSystemId: resources.filesystems[volume].ID(), SecurityGroups: resources.allSecurityGroups(), - SubnetId: subnet, + SubnetId: subnet.ID(), } } } @@ -40,7 +42,58 @@ func (b *ecsAPIService) createNFSMountTarget(project *types.Project, resources a func (b *ecsAPIService) mountTargets(volume string, resources awsResources) []string { var refs []string for _, subnet := range resources.subnets { - refs = append(refs, fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet))) + refs = append(refs, fmt.Sprintf("%sNFSMountTargetOn%s", normalizeResourceName(volume), normalizeResourceName(subnet.ID()))) } return refs } + +func (b *ecsAPIService) createAccessPoints(project *types.Project, r awsResources, template *cloudformation.Template) { + for name, volume := range project.Volumes { + n := fmt.Sprintf("%sAccessPoint", normalizeResourceName(name)) + + uid := volume.DriverOpts["uid"] + gid := volume.DriverOpts["gid"] + permissions := volume.DriverOpts["permissions"] + path := volume.DriverOpts["root_directory"] + + ap := efs.AccessPoint{ + AccessPointTags: []efs.AccessPoint_AccessPointTag{ + { + Key: compose.ProjectTag, + Value: project.Name, + }, + { + Key: compose.VolumeTag, + Value: name, + }, + { + Key: "Name", + Value: fmt.Sprintf("%s_%s", project.Name, name), + }, + }, + FileSystemId: r.filesystems[name].ID(), + } + + if uid != "" { + ap.PosixUser = &efs.AccessPoint_PosixUser{ + Uid: uid, + Gid: gid, + } + } + if path != "" { + root := efs.AccessPoint_RootDirectory{ + Path: path, + } + ap.RootDirectory = &root + if uid != "" { + root.CreationInfo = &efs.AccessPoint_CreationInfo{ + OwnerUid: uid, + OwnerGid: gid, + Permissions: permissions, + } + } + } + + template.Resources[n] = &ap + } +}