diff --git a/cli/main.go b/cli/main.go
index f4c52b2d..0153823a 100644
--- a/cli/main.go
+++ b/cli/main.go
@@ -47,7 +47,6 @@ import (
 	"github.com/docker/compose-cli/cli/mobycli"
 	cliopts "github.com/docker/compose-cli/cli/options"
 
-	cliconfig "github.com/docker/cli/cli/config"
 	cliflags "github.com/docker/cli/cli/flags"
 
 	// Backend registrations
@@ -214,12 +213,6 @@ func main() {
 		volume.Command(ctype),
 	)
 
-	configFile, err := cliconfig.Load(opts.Config)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Unable to load docker config: %s\n", opts.Config)
-		os.Exit(1)
-	}
-	ctx = apicontext.WithConfig(ctx, *configFile)
 	ctx = apicontext.WithCurrentContext(ctx, currentContext)
 
 	cnxOptions := cliflags.CommonOptions{
diff --git a/local/backend.go b/local/backend.go
index 5e2f042f..fa79f1eb 100644
--- a/local/backend.go
+++ b/local/backend.go
@@ -22,13 +22,17 @@ import (
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/compose-cli/api/backend"
 	"github.com/docker/compose-cli/api/cloud"
+
 	"github.com/docker/compose-cli/api/compose"
+	apiconfig "github.com/docker/compose-cli/api/config"
 	"github.com/docker/compose-cli/api/containers"
 	apicontext "github.com/docker/compose-cli/api/context"
 	"github.com/docker/compose-cli/api/resources"
 	"github.com/docker/compose-cli/api/secrets"
 	"github.com/docker/compose-cli/api/volumes"
 	local_compose "github.com/docker/compose-cli/local/compose"
+
+	cliconfig "github.com/docker/cli/cli/config"
 )
 
 type local struct {
@@ -43,9 +47,12 @@ func init() {
 
 func service(ctx context.Context) (backend.Service, error) {
 	options := apicontext.CliOptions(ctx)
-	config := apicontext.Config(ctx)
-
-	apiClient, err := command.NewAPIClientFromFlags(&options, &config)
+	config := apiconfig.Dir(ctx)
+	configFile, err := cliconfig.Load(config)
+	if err != nil {
+		return nil, err
+	}
+	apiClient, err := command.NewAPIClientFromFlags(&options, configFile)
 	if err != nil {
 		return nil, err
 	}
diff --git a/local/e2e/cli-only/e2e_test.go b/local/e2e/cli-only/e2e_test.go
index 40faf6f9..8b8f1b75 100644
--- a/local/e2e/cli-only/e2e_test.go
+++ b/local/e2e/cli-only/e2e_test.go
@@ -396,11 +396,11 @@ func TestLegacy(t *testing.T) {
 	})
 
 	t.Run("host flag", func(t *testing.T) {
-		stderr := "Cannot connect to the Docker daemon at tcp://localhost:123"
+		stderr := "nonexistent: Name or service not known"
 		if runtime.GOOS == "windows" {
-			stderr = "error during connect: Get http://localhost:123"
+			stderr = "error during connect: Get http://nonexitent:123"
 		}
-		res := c.RunDockerOrExitError("-H", "tcp://localhost:123", "version")
+		res := c.RunDockerOrExitError("-H", "tcp://nonexistent:123", "version")
 		res.Assert(t, icmd.Expected{
 			ExitCode: 1,
 			Err:      stderr,