Fix TAP output with multiple RuntimeInsideValidate

docs/devel_guidelines.md

@@ -7,9 +7,52 @@ Each validation test has a `.go` file in the `validation/` directory and can be
 ### TAP output
 
 Each validation test prints TAP output.
-So far, we have two kinds of validation tests and they print the TAP output differently:
-* tests using `util.RuntimeInsideValidate`: they start the process `runtimetest` inside the container and `runtimetest` prints the TAP output. The test process itself must not output anything to avoid mixing its output with the TAP output. Each test can only call `util.RuntimeInsideValidate` one time because several TAP outputs cannot be concatenated.
-* tests using `util.RuntimeOutsideValidate`: they create a container but without executing `runtimetest`. The test program itself must print the TAP output.
+So far, we have three kinds of validation tests and they print the TAP output differently:
+
+#### Using `util.RuntimeOutsideValidate`
+
+They create a container but without executing `runtimetest`. The test program itself must print the TAP output.
+
+Example:
+```go
+err = util.RuntimeOutsideValidate(g, t, func(config *rspec.Spec, t *tap.T, state *rspec.State) error {
+        err := testFoo()
+        t.Ok((err == nil), "check foo")
+        if err != nil {
+                t.Diagnostic(err.Error())
+                return nil
+        }
+        return nil
+})
+
+```
+#### Using `util.RuntimeInsideValidate` and passthrough
+
+They start the process `runtimetest` inside the container and `runtimetest` prints the TAP output.
+The test process itself must not output anything to avoid mixing its output with the TAP output.
+Each test can only call `util.RuntimeInsideValidate` one time because several TAP outputs cannot be concatenated.
+
+Example:
+```go
+err = util.RuntimeInsideValidate(g, nil, nil)
+if err != nil {
+        util.Fatal(err)
+}
+```
+
+#### Using `util.RuntimeInsideValidate` and encapsulation
+
+Similar to the passthrough variant but the test consumes the output from `runtimetest` and re-emit a single TAP result for the container run.
+For that, the TAP object must be passed as parameter to `util.RuntimeInsideValidate`.
+
+Example:
+```go
+g.AddAnnotation("TestName", "check foo")
+err = util.RuntimeInsideValidate(g, t, nil)
+if err != nil {
+        util.Fatal(err)
+}
+```
 
 ### Exit status
 

validation/default/default.go

@@ -9,7 +9,7 @@ func main() {
 	if err != nil {
 		util.Fatal(err)
 	}
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}

validation/hostname/hostname.go

@@ -15,7 +15,8 @@ func testHostname(t *tap.T, hostname string) error {
 	}
 
 	g.SetHostname(hostname)
-	err = util.RuntimeInsideValidate(g, nil)
+	g.AddAnnotation("TestName", fmt.Sprintf("check hostname %q", hostname))
+	err = util.RuntimeInsideValidate(g, t, nil)
 	t.Ok(err == nil, "hostname is set correctly")
 	if err != nil {
 		t.Diagnosticf("expect: err == nil, actual: err != nil")

validation/linux_devices/linux_devices.go

@@ -51,7 +51,7 @@ func main() {
 	pdev.FileMode = &pmode
 	g.AddDevice(pdev)
 
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}

validation/linux_masked_paths/linux_masked_paths.go

@@ -6,11 +6,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mndrix/tap-go"
 	"github.com/opencontainers/runtime-tools/validation/util"
 	"golang.org/x/sys/unix"
 )
 
-func checkMaskedPaths() error {
+func checkMaskedPaths(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -32,7 +33,8 @@ func checkMaskedPaths() error {
 	g.AddLinuxMaskedPaths(maskedDirSub)
 	g.AddLinuxMaskedPaths(maskedFileSub)
 	g.AddLinuxMaskedPaths(maskedFileSubSub)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check masked paths")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testDir := filepath.Join(path, maskedDirSub)
 		err = os.MkdirAll(testDir, 0777)
 		if err != nil {
@@ -63,7 +65,7 @@ func checkMaskedPaths() error {
 	return err
 }
 
-func checkMaskedRelPaths() error {
+func checkMaskedRelPaths(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -73,7 +75,8 @@ func checkMaskedRelPaths() error {
 	maskedRelPath := "masked-relpath"
 
 	g.AddLinuxMaskedPaths(maskedRelPath)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check masked relative paths")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, maskedRelPath)
 		if _, err := os.Stat(testFile); err != nil && os.IsNotExist(err) {
 			return err
@@ -87,7 +90,7 @@ func checkMaskedRelPaths() error {
 	return fmt.Errorf("expected: err != nil, actual: err == nil")
 }
 
-func checkMaskedSymlinks() error {
+func checkMaskedSymlinks(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -98,7 +101,8 @@ func checkMaskedSymlinks() error {
 	maskedSymlink := "/masked-symlink"
 
 	g.AddLinuxMaskedPaths(maskedSymlink)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check masked symlinks")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, maskedSymlink)
 		// ln -s .. /masked-symlink ; readlink -f /masked-symlink; ls -L /masked-symlink
 		if err := os.Symlink("../masked-symlink", testFile); err != nil {
@@ -121,7 +125,7 @@ func checkMaskedSymlinks() error {
 	return fmt.Errorf("expected: err != nil, actual: err == nil")
 }
 
-func checkMaskedDeviceNodes(mode uint32) error {
+func checkMaskedDeviceNodes(t *tap.T, mode uint32) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -130,7 +134,8 @@ func checkMaskedDeviceNodes(mode uint32) error {
 	maskedDevice := "/masked-device"
 
 	g.AddLinuxMaskedPaths(maskedDevice)
-	return util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check masked device nodes")
+	return util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, maskedDevice)
 
 		if err := unix.Mknod(testFile, mode, 0); err != nil {
@@ -146,15 +151,19 @@ func checkMaskedDeviceNodes(mode uint32) error {
 }
 
 func main() {
-	if err := checkMaskedPaths(); err != nil {
+	t := tap.New()
+	t.Header(0)
+	defer t.AutoPlan()
+
+	if err := checkMaskedPaths(t); err != nil {
 		util.Fatal(err)
 	}
 
-	if err := checkMaskedRelPaths(); err != nil {
+	if err := checkMaskedRelPaths(t); err != nil {
 		util.Fatal(err)
 	}
 
-	if err := checkMaskedSymlinks(); err != nil {
+	if err := checkMaskedSymlinks(t); err != nil {
 		util.Fatal(err)
 	}
 
@@ -167,7 +176,7 @@ func main() {
 	}
 
 	for _, m := range modes {
-		if err := checkMaskedDeviceNodes(m); err != nil {
+		if err := checkMaskedDeviceNodes(t, m); err != nil {
 			util.Fatal(err)
 		}
 	}

validation/linux_readonly_paths/linux_readonly_paths.go

@@ -6,11 +6,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mndrix/tap-go"
 	"github.com/opencontainers/runtime-tools/validation/util"
 	"golang.org/x/sys/unix"
 )
 
-func checkReadonlyPaths() error {
+func checkReadonlyPaths(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -32,7 +33,8 @@ func checkReadonlyPaths() error {
 	g.AddLinuxReadonlyPaths(readonlyDirSub)
 	g.AddLinuxReadonlyPaths(readonlyFileSub)
 	g.AddLinuxReadonlyPaths(readonlyFileSubSub)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check read-only paths")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testDir := filepath.Join(path, readonlyDirSub)
 		err = os.MkdirAll(testDir, 0777)
 		if err != nil {
@@ -63,7 +65,7 @@ func checkReadonlyPaths() error {
 	return err
 }
 
-func checkReadonlyRelPaths() error {
+func checkReadonlyRelPaths(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -73,7 +75,8 @@ func checkReadonlyRelPaths() error {
 	readonlyRelPath := "readonly-relpath"
 
 	g.AddLinuxReadonlyPaths(readonlyRelPath)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check read-only relative paths")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, readonlyRelPath)
 		if _, err := os.Stat(testFile); err != nil && os.IsNotExist(err) {
 			return err
@@ -87,7 +90,7 @@ func checkReadonlyRelPaths() error {
 	return fmt.Errorf("expected: err != nil, actual: err == nil")
 }
 
-func checkReadonlySymlinks() error {
+func checkReadonlySymlinks(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -98,7 +101,8 @@ func checkReadonlySymlinks() error {
 	readonlySymlink := "/readonly-symlink"
 
 	g.AddLinuxReadonlyPaths(readonlySymlink)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check read-only symlinks")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, readonlySymlink)
 		// ln -s .. /readonly-symlink ; readlink -f /readonly-symlink; ls -L /readonly-symlink
 		if err := os.Symlink("../readonly-symlink", testFile); err != nil {
@@ -121,7 +125,7 @@ func checkReadonlySymlinks() error {
 	return fmt.Errorf("expected: err != nil, actual: err == nil")
 }
 
-func checkReadonlyDeviceNodes(mode uint32) error {
+func checkReadonlyDeviceNodes(t *tap.T, mode uint32) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -130,7 +134,8 @@ func checkReadonlyDeviceNodes(mode uint32) error {
 	readonlyDevice := "/readonly-device"
 
 	g.AddLinuxReadonlyPaths(readonlyDevice)
-	return util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check read-only device nodes")
+	return util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, readonlyDevice)
 
 		if err := unix.Mknod(testFile, mode, 0); err != nil {
@@ -146,15 +151,19 @@ func checkReadonlyDeviceNodes(mode uint32) error {
 }
 
 func main() {
-	if err := checkReadonlyPaths(); err != nil {
+	t := tap.New()
+	t.Header(0)
+	defer t.AutoPlan()
+
+	if err := checkReadonlyPaths(t); err != nil {
 		util.Fatal(err)
 	}
 
-	if err := checkReadonlyRelPaths(); err != nil {
+	if err := checkReadonlyRelPaths(t); err != nil {
 		util.Fatal(err)
 	}
 
-	if err := checkReadonlySymlinks(); err != nil {
+	if err := checkReadonlySymlinks(t); err != nil {
 		util.Fatal(err)
 	}
 
@@ -167,7 +176,7 @@ func main() {
 	}
 
 	for _, m := range modes {
-		if err := checkReadonlyDeviceNodes(m); err != nil {
+		if err := checkReadonlyDeviceNodes(t, m); err != nil {
 			util.Fatal(err)
 		}
 	}

validation/linux_rootfs_propagation/linux_rootfs_propagation.go

@@ -5,14 +5,15 @@ import (
 	"github.com/opencontainers/runtime-tools/validation/util"
 )
 
-func testLinuxRootPropagation(propMode string) error {
+func testLinuxRootPropagation(t *tap.T, propMode string) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		util.Fatal(err)
 	}
 	g.SetupPrivileged(true)
 	g.SetLinuxRootPropagation(propMode)
-	return util.RuntimeInsideValidate(g, nil)
+	g.AddAnnotation("TestName", "check root propagation")
+	return util.RuntimeInsideValidate(g, t, nil)
 }
 
 func main() {
@@ -28,7 +29,7 @@ func main() {
 	}
 
 	for _, c := range cases {
-		if err := testLinuxRootPropagation(c); err != nil {
+		if err := testLinuxRootPropagation(t, c); err != nil {
 			t.Fail(err.Error())
 		}
 	}

validation/linux_seccomp/linux_seccomp.go

@@ -16,7 +16,7 @@ func main() {
 	}
 	g.SetDefaultSeccompAction("allow")
 	g.SetSyscallAction(syscallArgs)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}

validation/linux_sysctl/linux_sysctl.go

@@ -10,7 +10,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.AddLinuxSysctl("net.ipv4.ip_forward", "1")
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}

validation/linux_uid_mappings/linux_uid_mappings.go

@@ -12,7 +12,7 @@ func main() {
 	g.AddOrReplaceLinuxNamespace("user", "")
 	g.AddLinuxUIDMapping(uint32(1000), uint32(0), uint32(2000))
 	g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3000))
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}

validation/mounts/mounts.go

@@ -22,7 +22,7 @@ func main() {
 		},
 	}
 	g.AddMount(mount)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}

validation/process/process.go

@@ -16,7 +16,7 @@ func main() {
 	g.AddProcessEnv("testa", "valuea")
 	g.AddProcessEnv("testb", "123")
 
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		pathName := filepath.Join(path, "test")
 		return os.MkdirAll(pathName, 0700)
 	})

validation/process_capabilities/process_capabilities.go

@@ -18,7 +18,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.SetupPrivileged(true)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}

validation/process_capabilities_fail/process_capabilities_fail.go

@@ -21,7 +21,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.AddProcessCapabilityBounding("CAP_TEST")
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err == nil {
 		util.Fatal(specerror.NewError(specerror.LinuxProcCapError, fmt.Errorf("Any value which cannot be mapped to a relevant kernel interface MUST cause an error"), rspecs.Version))
 	}