Add e2e tests for external agents

.github/workflows/e2e-isolated.yml

@@ -8,7 +8,7 @@ on:
         required: true
         default: "gemini-cli"
         type: choice
-        options: [claude-code, opencode, gemini-cli, cursor-cli, factoryai-droid, copilot-cli]
+        options: [claude-code, opencode, gemini-cli, cursor-cli, factoryai-droid, copilot-cli, roger-roger]
       test:
         description: "Test name filter (regex)"
         required: true
@@ -41,10 +41,14 @@ jobs:
             cursor-cli)  curl https://cursor.com/install -fsS | bash ;;
             factoryai-droid) curl -fsSL https://app.factory.ai/cli | sh ;;
             copilot-cli)  npm install -g @github/copilot  ;;
+            roger-roger)
+              GOBIN=/usr/local/bin go install github.com/entireio/roger-roger/cmd/roger-roger@latest github.com/entireio/roger-roger/cmd/entire-agent-roger-roger@latest
+              ;;
           esac
           echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Bootstrap agent
+        if: inputs.agent != 'roger-roger'
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}

.github/workflows/e2e.yml

@@ -15,6 +15,7 @@ on:
           - factoryai-droid
           - cursor-cli
           - copilot-cli
+          - roger-roger
   push:
     branches:
       - main
@@ -41,7 +42,7 @@ jobs:
           if [ -n "$input" ]; then
             echo "agents=[\"$input\"]" >> "$GITHUB_OUTPUT"
           else
-            echo 'agents=["claude-code","opencode","gemini-cli","factoryai-droid","cursor-cli","copilot-cli"]' >> "$GITHUB_OUTPUT"
+            echo 'agents=["claude-code","opencode","gemini-cli","factoryai-droid","cursor-cli","copilot-cli","roger-roger"]' >> "$GITHUB_OUTPUT"
           fi
 
   e2e-tests:
@@ -72,10 +73,14 @@ jobs:
             cursor-cli)  curl https://cursor.com/install -fsS | bash ;;
             factoryai-droid) curl -fsSL https://app.factory.ai/cli | sh ;;
             copilot-cli)  npm install -g @github/copilot  ;;
+            roger-roger)
+              GOBIN=/usr/local/bin go install github.com/entireio/roger-roger/cmd/roger-roger@latest github.com/entireio/roger-roger/cmd/entire-agent-roger-roger@latest
+              ;;
           esac
           echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Bootstrap agent
+        if: matrix.agent != 'roger-roger'
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
@@ -92,7 +97,7 @@ jobs:
           FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }}
           COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
           E2E_CONCURRENT_TEST_LIMIT: ${{ matrix.agent == 'gemini-cli' && '6' || matrix.agent == 'factoryai-droid' && '1' || '' }}
-        run: mise run test:e2e --agent ${{ matrix.agent }}
+        run: mise run test:e2e --agent ${{ matrix.agent }} ${{ matrix.agent == 'roger-roger' && 'TestExternalAgent' || '' }}
 
       - name: Upload artifacts
         if: always()

.gitignore

@@ -38,6 +38,7 @@ go.work.sum
 /entire
 /vogon
 /testreport
+/bin
 
 # Build output directory
 /dist/

e2e/agents/agent.go

@@ -61,6 +61,13 @@ type Session interface {
 	Close() error
 }
 
+// ExternalAgent is an optional interface for agents discovered via the
+// external agent protocol (entire-agent-* binaries). SetupRepo uses this
+// to pre-configure external_agents in settings before running `entire enable`.
+type ExternalAgent interface {
+	IsExternalAgent() bool
+}
+
 var registry []Agent
 var gates = map[string]chan struct{}{}
 

e2e/agents/roger_roger.go

@@ -0,0 +1,96 @@
+package agents
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+)
+
+func init() {
+	if env := os.Getenv("E2E_AGENT"); env != "" && env != "roger-roger" {
+		return
+	}
+	// Only register if both binaries exist (roger-roger REPL + protocol handler).
+	if _, err := exec.LookPath("roger-roger"); err != nil {
+		return
+	}
+	if _, err := exec.LookPath("entire-agent-roger-roger"); err != nil {
+		return
+	}
+	Register(&RogerRoger{})
+}
+
+// RogerRoger implements the Agent interface using a deterministic external
+// agent binary that creates files and fires hooks without making any API calls.
+// Unlike built-in agents (Vogon, Claude Code, etc.), roger-roger is discovered
+// via the external agent protocol (entire-agent-roger-roger binary in $PATH).
+type RogerRoger struct{}
+
+func (r *RogerRoger) Name() string                            { return "roger-roger" }
+func (r *RogerRoger) Binary() string                          { return "roger-roger" }
+func (r *RogerRoger) EntireAgent() string                     { return "roger-roger" }
+func (r *RogerRoger) PromptPattern() string                   { return `>` }
+func (r *RogerRoger) TimeoutMultiplier() float64              { return 0.5 } // Deterministic, no API calls
+func (r *RogerRoger) Bootstrap() error                        { return nil }
+func (r *RogerRoger) IsTransientError(_ Output, _ error) bool { return false }
+
+// IsExternalAgent returns true — roger-roger is discovered via the external agent protocol.
+func (r *RogerRoger) IsExternalAgent() bool { return true }
+
+func (r *RogerRoger) RunPrompt(ctx context.Context, dir string, prompt string, opts ...Option) (Output, error) {
+	// roger-roger reads prompts from stdin line by line.
+	// An empty line causes the REPL to exit gracefully.
+	cmd := exec.CommandContext(ctx, r.Binary())
+	cmd.Dir = dir
+	cmd.Stdin = strings.NewReader(prompt + "\n\n")
+	cmd.Env = filterEnv(os.Environ(), "ENTIRE_TEST_TTY")
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	cmd.Cancel = func() error {
+		return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+	}
+	cmd.WaitDelay = 5 * time.Second
+
+	var stdout, stderr strings.Builder
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	exitCode := 0
+	if err != nil {
+		exitErr := &exec.ExitError{}
+		if errors.As(err, &exitErr) {
+			exitCode = exitErr.ExitCode()
+		} else {
+			exitCode = -1
+		}
+	}
+
+	return Output{
+		Command:  "roger-roger <<< " + fmt.Sprintf("%q", prompt),
+		Stdout:   stdout.String(),
+		Stderr:   stderr.String(),
+		ExitCode: exitCode,
+	}, err
+}
+
+func (r *RogerRoger) StartSession(_ context.Context, dir string) (Session, error) {
+	name := fmt.Sprintf("rr-test-%d", time.Now().UnixNano())
+	s, err := NewTmuxSession(name, dir, []string{"ENTIRE_TEST_TTY"}, r.Binary())
+	if err != nil {
+		return nil, err
+	}
+
+	// Wait for the interactive prompt.
+	if _, err := s.WaitFor(`>`, 10*time.Second); err != nil {
+		_ = s.Close()
+		return nil, fmt.Errorf("waiting for startup prompt: %w", err)
+	}
+	s.stableAtSend = ""
+
+	return s, nil
+}

e2e/tests/external_agent_test.go

@@ -0,0 +1,137 @@
+//go:build e2e
+
+package tests
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/entireio/cli/e2e/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestExternalAgentSingleSessionManualCommit: external agent creates a file,
+// user commits manually. Validates the full external agent protocol flow:
+// discovery via PATH, hook firing through binary subcommands, transcript
+// handling, and checkpoint creation.
+func TestExternalAgentSingleSessionManualCommit(t *testing.T) {
+	testutil.ForEachAgent(t, 2*time.Minute, func(t *testing.T, s *testutil.RepoState, ctx context.Context) {
+		_, err := s.RunPrompt(t, ctx,
+			"create a file called docs/hello.md")
+		if err != nil {
+			t.Fatalf("agent failed: %v", err)
+		}
+
+		testutil.AssertFileExists(t, s.Dir, "docs/hello.md")
+
+		s.Git(t, "add", ".")
+		s.Git(t, "commit", "-m", "Add hello file via external agent")
+
+		testutil.AssertNewCommits(t, s, 1)
+		testutil.WaitForCheckpoint(t, s, 15*time.Second)
+		testutil.AssertCheckpointAdvanced(t, s)
+
+		cpID := testutil.AssertHasCheckpointTrailer(t, s.Dir, "HEAD")
+		testutil.AssertCheckpointExists(t, s.Dir, cpID)
+		testutil.AssertCheckpointMetadataComplete(t, s.Dir, cpID)
+		testutil.AssertCheckpointHasSingleSession(t, s.Dir, cpID)
+		testutil.AssertCheckpointFilesTouchedContains(t, s.Dir, cpID, "docs/hello.md")
+		testutil.AssertNoShadowBranches(t, s.Dir)
+	})
+}
+
+// TestExternalAgentMultipleTurnsManualCommit: external agent handles two
+// sequential prompts, user commits once. Both prompts should be captured
+// in a single checkpoint.
+func TestExternalAgentMultipleTurnsManualCommit(t *testing.T) {
+	testutil.ForEachAgent(t, 2*time.Minute, func(t *testing.T, s *testutil.RepoState, ctx context.Context) {
+		if !s.IsExternalAgent() {
+			t.Skip("skipping external agent test for non-external agent")
+		}
+
+		_, err := s.RunPrompt(t, ctx,
+			"create a file called src/alpha.txt")
+		if err != nil {
+			t.Fatalf("first prompt failed: %v", err)
+		}
+
+		_, err = s.RunPrompt(t, ctx,
+			"create a file called src/beta.txt")
+		if err != nil {
+			t.Fatalf("second prompt failed: %v", err)
+		}
+
+		testutil.AssertFileExists(t, s.Dir, "src/alpha.txt")
+		testutil.AssertFileExists(t, s.Dir, "src/beta.txt")
+
+		s.Git(t, "add", ".")
+		s.Git(t, "commit", "-m", "Add alpha and beta via external agent")
+
+		testutil.AssertNewCommits(t, s, 1)
+		testutil.WaitForCheckpoint(t, s, 15*time.Second)
+		testutil.AssertCheckpointAdvanced(t, s)
+
+		cpID := testutil.AssertHasCheckpointTrailer(t, s.Dir, "HEAD")
+		testutil.AssertCheckpointExists(t, s.Dir, cpID)
+		testutil.AssertCheckpointMetadataComplete(t, s.Dir, cpID)
+		testutil.AssertNoShadowBranches(t, s.Dir)
+
+		// Both files should appear in files_touched
+		testutil.AssertCheckpointFilesTouchedContains(t, s.Dir, cpID, "src/alpha.txt")
+		testutil.AssertCheckpointFilesTouchedContains(t, s.Dir, cpID, "src/beta.txt")
+	})
+}
+
+// TestExternalAgentDeepCheckpointValidation: verifies transcript content,
+// content hash, and prompt text are correctly captured through the external
+// agent protocol.
+func TestExternalAgentDeepCheckpointValidation(t *testing.T) {
+	testutil.ForEachAgent(t, 2*time.Minute, func(t *testing.T, s *testutil.RepoState, ctx context.Context) {
+		_, err := s.RunPrompt(t, ctx,
+			"create a file called notes/deep.md")
+		if err != nil {
+			t.Fatalf("agent failed: %v", err)
+		}
+
+		testutil.AssertFileExists(t, s.Dir, "notes/deep.md")
+
+		s.Git(t, "add", ".")
+		s.Git(t, "commit", "-m", "Deep checkpoint validation test")
+
+		testutil.WaitForCheckpoint(t, s, 15*time.Second)
+
+		cpID := testutil.AssertHasCheckpointTrailer(t, s.Dir, "HEAD")
+
+		testutil.ValidateCheckpointDeep(t, s.Dir, testutil.DeepCheckpointValidation{
+			CheckpointID:    cpID,
+			Strategy:        "manual-commit",
+			FilesTouched:    []string{"notes/deep.md"},
+			ExpectedPrompts: []string{"create a file called notes/deep.md"},
+		})
+	})
+}
+
+// TestExternalAgentSessionMetadata: verifies that checkpoint session metadata
+// correctly identifies the external agent type.
+func TestExternalAgentSessionMetadata(t *testing.T) {
+	testutil.ForEachAgent(t, 2*time.Minute, func(t *testing.T, s *testutil.RepoState, ctx context.Context) {
+		_, err := s.RunPrompt(t, ctx,
+			"create a file called meta/test.md")
+		if err != nil {
+			t.Fatalf("agent failed: %v", err)
+		}
+
+		s.Git(t, "add", ".")
+		s.Git(t, "commit", "-m", "Metadata test commit")
+
+		testutil.WaitForCheckpoint(t, s, 15*time.Second)
+
+		cpID := testutil.AssertHasCheckpointTrailer(t, s.Dir, "HEAD")
+
+		// Verify session metadata has a non-empty agent field
+		sm := testutil.ReadSessionMetadata(t, s.Dir, cpID, 0)
+		assert.NotEmpty(t, sm.Agent, "session metadata should have agent field set")
+		assert.NotEmpty(t, sm.SessionID, "session metadata should have session_id")
+	})
+}

e2e/testutil/repo.go

@@ -66,6 +66,19 @@ func SetupRepo(t *testing.T, agent agents.Agent) *RepoState {
 	Git(t, dir, "config", "user.email", "e2e@test.local")
 	Git(t, dir, "commit", "--allow-empty", "-m", "initial commit")
 
+	// External agents need external_agents enabled in settings before enable,
+	// so the CLI can discover the agent binary via PATH during DiscoverAndRegister.
+	if ea, ok := agent.(agents.ExternalAgent); ok && ea.IsExternalAgent() {
+		entireDir := filepath.Join(dir, ".entire")
+		if err := os.MkdirAll(entireDir, 0o755); err != nil {
+			t.Fatalf("create .entire for external agent: %v", err)
+		}
+		if err := os.WriteFile(filepath.Join(entireDir, "settings.json"),
+			[]byte("{\"external_agents\": true}\n"), 0o644); err != nil {
+			t.Fatalf("write external_agents setting: %v", err)
+		}
+	}
+
 	entire.Enable(t, dir, agent.EntireAgent())
 	if agent.Name() == "factoryai-droid" {
 		if err := configureDroidRepoSettings(dir); err != nil {
@@ -399,6 +412,13 @@ func (s *RepoState) WaitFor(t *testing.T, session agents.Session, pattern string
 	}
 }
 
+// IsExternalAgent returns true if the agent implements the ExternalAgent
+// interface and reports itself as external.
+func (s *RepoState) IsExternalAgent() bool {
+	ea, ok := s.Agent.(agents.ExternalAgent)
+	return ok && ea.IsExternalAgent()
+}
+
 // Send sends input to an interactive session and logs it to ConsoleLog.
 // Fails the test on error.
 func (s *RepoState) Send(t *testing.T, session agents.Session, input string) {

mise-tasks/test/e2e/roger-roger

@@ -0,0 +1,37 @@
+#!/bin/sh
+#MISE description="Run E2E tests with roger-roger external agent (no API calls)"
+#USAGE arg "[filter]" help="Test name filter (regex)" default=""
+
+set -eu
+
+export E2E_AGENT="roger-roger"
+
+# Build entire CLI
+mise run build
+export E2E_ENTIRE_BIN="$PWD/entire"
+
+# Install roger-roger binaries from GitHub
+ROGER_ROGER_MODULE="${ROGER_ROGER_MODULE:-github.com/entireio/roger-roger}"
+echo "installing roger-roger binaries from $ROGER_ROGER_MODULE"
+GOBIN="$PWD/bin" go install "$ROGER_ROGER_MODULE/cmd/roger-roger@latest" "$ROGER_ROGER_MODULE/cmd/entire-agent-roger-roger@latest"
+export PATH="$PWD/bin:$PATH"
+
+if [ -z "${E2E_ARTIFACT_DIR:-}" ]; then
+  E2E_ARTIFACT_DIR="$PWD/e2e/artifacts/roger-roger-$(date +%Y-%m-%dT%H-%M-%S)"
+fi
+export E2E_ARTIFACT_DIR
+mkdir -p "$E2E_ARTIFACT_DIR"
+echo "artifacts: $E2E_ARTIFACT_DIR"
+
+filter="${usage_filter:-}"
+
+gotestsum --format dots --jsonfile "$E2E_ARTIFACT_DIR/test-events.json" -- \
+  -tags=e2e -count=1 -timeout=10m \
+  ${filter:+-run "$filter"} \
+  ./e2e/tests/... || rc=$?
+
+go run ./e2e/cmd/testreport -color -o "$E2E_ARTIFACT_DIR/report.txt" "$E2E_ARTIFACT_DIR/test-events.json"
+echo ""
+cat "$E2E_ARTIFACT_DIR/entire-version.txt" 2>/dev/null
+echo "artifacts: $E2E_ARTIFACT_DIR"
+exit "${rc:-0}"