From 2494de00799dec063952bcdec7569cd513b2bb4b Mon Sep 17 00:00:00 2001
From: Charlie Moog <moogcharlie@gmail.com>
Date: Wed, 28 Oct 2020 01:08:59 -0500
Subject: [PATCH 1/2] Add env create|edit integration tests

---
 ci/integration/envs_test.go        | 100 +++++++++++++++++++++--------
 ci/integration/integration_test.go |   2 +-
 go.mod                             |   1 +
 3 files changed, 76 insertions(+), 27 deletions(-)

diff --git a/ci/integration/envs_test.go b/ci/integration/envs_test.go
index a4f75480..af710fad 100644
--- a/ci/integration/envs_test.go
+++ b/ci/integration/envs_test.go
@@ -2,28 +2,23 @@ package integration
 
 import (
 	"context"
+	"fmt"
+	"math"
 	"regexp"
 	"testing"
 
 	"cdr.dev/coder-cli/ci/tcli"
+	"cdr.dev/coder-cli/coder-sdk"
+	"cdr.dev/slog/sloggers/slogtest/assert"
+	"github.com/google/go-cmp/cmp"
 )
 
-// From Coder organization images
-// const ubuntuImgID = "5f443b16-30652892427b955601330fa5"
-
 func TestEnvsCLI(t *testing.T) {
 	t.Parallel()
 
 	run(t, "coder-cli-env-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
 		headlessLogin(ctx, t, c)
 
-		// Ensure binary is present.
-		c.Run(ctx, "which coder").Assert(t,
-			tcli.Success(),
-			tcli.StdoutMatches("/usr/sbin/coder"),
-			tcli.StderrEmpty(),
-		)
-
 		// Minimum args not received.
 		c.Run(ctx, "coder envs create").Assert(t,
 			tcli.StderrMatches(regexp.QuoteMeta("accepts 1 arg(s), received 0")),
@@ -49,21 +44,74 @@ func TestEnvsCLI(t *testing.T) {
 			tcli.Error(),
 		)
 
-		// TODO(Faris) : uncomment this when we can safely purge the environments
-		// the integrations tests would create in the sidecar
-		// Successfully create environment.
-		// c.Run(ctx, "coder envs create --image "+ubuntuImgID+" test-ubuntu").Assert(t,
-		// 	tcli.Success(),
-		// 	// why does flog.Success write to stderr?
-		// 	tcli.StderrMatches(regexp.QuoteMeta("Successfully created environment \"test-ubuntu\"")),
-		// )
-
-		// TODO(Faris) : uncomment this when we can safely purge the environments
-		// the integrations tests would create in the sidecar
-		// Successfully provision environment with fractional resource amounts
-		// c.Run(ctx, fmt.Sprintf(`coder envs create -i %s -c 1.2 -m 1.4 non-whole-resource-amounts`, ubuntuImgID)).Assert(t,
-		// 	tcli.Success(),
-		// 	tcli.StderrMatches(regexp.QuoteMeta("Successfully created environment \"non-whole-resource-amounts\"")),
-		// )
+		name := randString(10)
+		cpu := 2.3
+		c.Run(ctx, fmt.Sprintf("coder envs create %s --image ubuntu --cpu %f", name, cpu)).Assert(t,
+			tcli.Success(),
+		)
+
+		t.Cleanup(func() {
+			run(t, "coder-envs-edit-cleanup", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
+				headlessLogin(ctx, t, c)
+				c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t)
+			})
+		})
+
+		c.Run(ctx, "coder envs ls").Assert(t,
+			tcli.Success(),
+			tcli.StdoutMatches(regexp.QuoteMeta(name)),
+		)
+
+		var env coder.Environment
+		c.Run(ctx, fmt.Sprintf(`coder envs ls -o json | jq '.[] | select(.name == "%s")'`, name)).Assert(t,
+			tcli.Success(),
+			tcli.StdoutJSONUnmarshal(&env),
+		)
+		assert.Equal(t, "environment cpu was correctly set", cpu, float64(env.CPUCores), floatComparer)
+
+		c.Run(ctx, fmt.Sprintf("coder envs watch-build %s", name)).Assert(t,
+			tcli.Success(),
+		)
+
+		c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t,
+			tcli.Success(),
+		)
+	})
+
+	run(t, "coder-cli-env-edit-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
+		headlessLogin(ctx, t, c)
+
+		name := randString(10)
+		c.Run(ctx, fmt.Sprintf("coder envs create %s --image ubuntu --follow", name)).Assert(t,
+			tcli.Success(),
+		)
+		t.Cleanup(func() {
+			run(t, "coder-envs-edit-cleanup", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
+				headlessLogin(ctx, t, c)
+				c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t)
+			})
+		})
+
+		cpu := 2.1
+		c.Run(ctx, fmt.Sprintf(`coder envs edit %s --cpu %f --follow`, name, cpu)).Assert(t,
+			tcli.Success(),
+		)
+
+		var env coder.Environment
+		c.Run(ctx, fmt.Sprintf(`coder envs ls -o json | jq '.[] | select(.name == "%s")'`, name)).Assert(t,
+			tcli.Success(),
+			tcli.StdoutJSONUnmarshal(&env),
+		)
+		assert.Equal(t, "cpu cores were updated", cpu, float64(env.CPUCores), floatComparer)
+
+		c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t,
+			tcli.Success(),
+		)
 	})
 }
+
+var floatComparer = cmp.Comparer(func(x, y float64) bool {
+	delta := math.Abs(x - y)
+	mean := math.Abs(x+y) / 2.0
+	return delta/mean < 0.001
+})
diff --git a/ci/integration/integration_test.go b/ci/integration/integration_test.go
index eccf68a2..7feb4f47 100644
--- a/ci/integration/integration_test.go
+++ b/ci/integration/integration_test.go
@@ -87,7 +87,7 @@ func TestCoderCLI(t *testing.T) {
 var seededRand = rand.New(rand.NewSource(time.Now().UnixNano()))
 
 func randString(length int) string {
-	const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	const charset = "abcdefghijklmnopqrstuvwxyz"
 	b := make([]byte, length)
 	for i := range b {
 		b[i] = charset[seededRand.Intn(len(charset))]
diff --git a/go.mod b/go.mod
index 6c90ab40..cbc0fd04 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
 	cdr.dev/wsep v0.0.0-20200728013649-82316a09813f
 	github.com/briandowns/spinner v1.11.1
 	github.com/fatih/color v1.9.0
+	github.com/google/go-cmp v0.4.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
 	github.com/klauspost/compress v1.10.8 // indirect

From 84f51adcc6596650e48e09354e9f87feeef8f5a9 Mon Sep 17 00:00:00 2001
From: Charlie Moog <moogcharlie@gmail.com>
Date: Thu, 29 Oct 2020 22:27:20 -0500
Subject: [PATCH 2/2] Use smaller image for integration tests

---
 .github/workflows/integration.yaml |  4 +--
 ci/integration/Dockerfile          |  3 ++
 ci/integration/devurls_test.go     |  1 -
 ci/integration/envs_test.go        | 52 ++++++++++++++++++++++--------
 ci/integration/integration_test.go |  3 +-
 ci/integration/setup_test.go       |  7 ++--
 ci/integration/users_test.go       |  1 -
 ci/steps/integration.sh            | 15 +++++++++
 8 files changed, 64 insertions(+), 22 deletions(-)
 create mode 100644 ci/integration/Dockerfile
 create mode 100755 ci/steps/integration.sh

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index a8045e43..8ed56350 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -22,5 +22,5 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: '^1.14'
-      - name: go test
-        run: go test -v ./ci/integration/...
+      - name: integration tests
+        run: ./ci/steps/integration.sh
diff --git a/ci/integration/Dockerfile b/ci/integration/Dockerfile
new file mode 100644
index 00000000..f81aa240
--- /dev/null
+++ b/ci/integration/Dockerfile
@@ -0,0 +1,3 @@
+FROM ubuntu:20.04
+
+RUN apt-get update && apt-get install -y jq curl
diff --git a/ci/integration/devurls_test.go b/ci/integration/devurls_test.go
index 06553107..6dee8211 100644
--- a/ci/integration/devurls_test.go
+++ b/ci/integration/devurls_test.go
@@ -12,7 +12,6 @@ func TestDevURLCLI(t *testing.T) {
 	run(t, "coder-cli-devurl-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
 		c.Run(ctx, "which coder").Assert(t,
 			tcli.Success(),
-			tcli.StdoutMatches("/usr/sbin/coder"),
 			tcli.StderrEmpty(),
 		)
 
diff --git a/ci/integration/envs_test.go b/ci/integration/envs_test.go
index af710fad..075f7222 100644
--- a/ci/integration/envs_test.go
+++ b/ci/integration/envs_test.go
@@ -4,20 +4,44 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"net/url"
 	"regexp"
 	"testing"
+	"time"
 
 	"cdr.dev/coder-cli/ci/tcli"
 	"cdr.dev/coder-cli/coder-sdk"
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
 	"cdr.dev/slog/sloggers/slogtest/assert"
 	"github.com/google/go-cmp/cmp"
 )
 
+func cleanupClient(t *testing.T, ctx context.Context) *coder.Client {
+	creds := login(ctx, t)
+
+	u, err := url.Parse(creds.url)
+	assert.Success(t, "parse base url", err)
+
+	return &coder.Client{BaseURL: u, Token: creds.token}
+}
+
+func cleanupEnv(t *testing.T, client *coder.Client, envID string) func() {
+	return func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+
+		slogtest.Info(t, "cleanuping up environment", slog.F("env_id", envID))
+		_ = client.DeleteEnvironment(ctx, envID)
+	}
+}
+
 func TestEnvsCLI(t *testing.T) {
 	t.Parallel()
 
 	run(t, "coder-cli-env-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
 		headlessLogin(ctx, t, c)
+		client := cleanupClient(t, ctx)
 
 		// Minimum args not received.
 		c.Run(ctx, "coder envs create").Assert(t,
@@ -50,13 +74,6 @@ func TestEnvsCLI(t *testing.T) {
 			tcli.Success(),
 		)
 
-		t.Cleanup(func() {
-			run(t, "coder-envs-edit-cleanup", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
-				headlessLogin(ctx, t, c)
-				c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t)
-			})
-		})
-
 		c.Run(ctx, "coder envs ls").Assert(t,
 			tcli.Success(),
 			tcli.StdoutMatches(regexp.QuoteMeta(name)),
@@ -67,6 +84,10 @@ func TestEnvsCLI(t *testing.T) {
 			tcli.Success(),
 			tcli.StdoutJSONUnmarshal(&env),
 		)
+
+		// attempt to cleanup the environment even if tests fail
+		t.Cleanup(cleanupEnv(t, client, env.ID))
+
 		assert.Equal(t, "environment cpu was correctly set", cpu, float64(env.CPUCores), floatComparer)
 
 		c.Run(ctx, fmt.Sprintf("coder envs watch-build %s", name)).Assert(t,
@@ -80,24 +101,27 @@ func TestEnvsCLI(t *testing.T) {
 
 	run(t, "coder-cli-env-edit-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
 		headlessLogin(ctx, t, c)
+		client := cleanupClient(t, ctx)
 
 		name := randString(10)
 		c.Run(ctx, fmt.Sprintf("coder envs create %s --image ubuntu --follow", name)).Assert(t,
 			tcli.Success(),
 		)
-		t.Cleanup(func() {
-			run(t, "coder-envs-edit-cleanup", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
-				headlessLogin(ctx, t, c)
-				c.Run(ctx, fmt.Sprintf("coder envs rm %s --force", name)).Assert(t)
-			})
-		})
+
+		var env coder.Environment
+		c.Run(ctx, fmt.Sprintf(`coder envs ls -o json | jq '.[] | select(.name == "%s")'`, name)).Assert(t,
+			tcli.Success(),
+			tcli.StdoutJSONUnmarshal(&env),
+		)
+
+		// attempt to cleanup the environment even if tests fail
+		t.Cleanup(cleanupEnv(t, client, env.ID))
 
 		cpu := 2.1
 		c.Run(ctx, fmt.Sprintf(`coder envs edit %s --cpu %f --follow`, name, cpu)).Assert(t,
 			tcli.Success(),
 		)
 
-		var env coder.Environment
 		c.Run(ctx, fmt.Sprintf(`coder envs ls -o json | jq '.[] | select(.name == "%s")'`, name)).Assert(t,
 			tcli.Success(),
 			tcli.StdoutJSONUnmarshal(&env),
diff --git a/ci/integration/integration_test.go b/ci/integration/integration_test.go
index 7feb4f47..52527839 100644
--- a/ci/integration/integration_test.go
+++ b/ci/integration/integration_test.go
@@ -18,7 +18,7 @@ func run(t *testing.T, container string, execute func(t *testing.T, ctx context.
 		defer cancel()
 
 		c, err := tcli.NewContainerRunner(ctx, &tcli.ContainerConfig{
-			Image: "codercom/enterprise-dev",
+			Image: "coder-cli-integration:latest",
 			Name:  container,
 			BindMounts: map[string]string{
 				binpath: "/bin/coder",
@@ -36,7 +36,6 @@ func TestCoderCLI(t *testing.T) {
 	run(t, "test-coder-cli", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
 		c.Run(ctx, "which coder").Assert(t,
 			tcli.Success(),
-			tcli.StdoutMatches("/usr/sbin/coder"),
 			tcli.StderrEmpty(),
 		)
 
diff --git a/ci/integration/setup_test.go b/ci/integration/setup_test.go
index 566f40e9..1919f6a0 100644
--- a/ci/integration/setup_test.go
+++ b/ci/integration/setup_test.go
@@ -51,14 +51,17 @@ func build(path string) error {
 // write session tokens to the given container runner
 func headlessLogin(ctx context.Context, t *testing.T, runner *tcli.ContainerRunner) {
 	creds := login(ctx, t)
-	cmd := exec.CommandContext(ctx, "sh", "-c", "mkdir -p ~/.config/coder && cat > ~/.config/coder/session")
+	cmd := exec.CommandContext(ctx, "sh", "-c", "mkdir -p $HOME/.config/coder && cat > $HOME/.config/coder/session")
 
 	// !IMPORTANT: be careful that this does not appear in logs
 	cmd.Stdin = strings.NewReader(creds.token)
 	runner.RunCmd(cmd).Assert(t,
 		tcli.Success(),
 	)
-	runner.Run(ctx, fmt.Sprintf("echo -ne %s > ~/.config/coder/url", creds.url)).Assert(t,
+
+	cmd = exec.CommandContext(ctx, "sh", "-c", "cat > $HOME/.config/coder/url")
+	cmd.Stdin = strings.NewReader(creds.url)
+	runner.RunCmd(cmd).Assert(t,
 		tcli.Success(),
 	)
 }
diff --git a/ci/integration/users_test.go b/ci/integration/users_test.go
index 2138aeed..55d554f2 100644
--- a/ci/integration/users_test.go
+++ b/ci/integration/users_test.go
@@ -14,7 +14,6 @@ func TestUsers(t *testing.T) {
 	run(t, "users-cli-tests", func(t *testing.T, ctx context.Context, c *tcli.ContainerRunner) {
 		c.Run(ctx, "which coder").Assert(t,
 			tcli.Success(),
-			tcli.StdoutMatches("/usr/sbin/coder"),
 			tcli.StderrEmpty(),
 		)
 
diff --git a/ci/steps/integration.sh b/ci/steps/integration.sh
new file mode 100755
index 00000000..1ef04178
--- /dev/null
+++ b/ci/steps/integration.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eo pipefail
+
+log() {
+  echo "--- $@"
+}
+
+cd "$(git rev-parse --show-toplevel)"
+
+log "building integration test image"
+docker build -f ./ci/integration/Dockerfile -t coder-cli-integration:latest .
+
+log "starting integration tests"
+go test ./ci/integration -count=1
