validation-tests: fix several tests

Makefile

@@ -58,7 +58,10 @@ validation-executables: $(VALIDATION_TESTS)
 $(VALIDATION_TESTS): %.t: %.go
 	go build -tags "$(BUILDTAGS)" ${TESTFLAGS} -o $@ $<
 
-.PHONY: test .gofmt .govet .golint
+print-validation-tests:
+	@echo $(VALIDATION_TESTS)
+
+.PHONY: test .gofmt .govet .golint print-validation-tests
 
 PACKAGES = $(shell go list ./... | grep -v vendor)
 test: .gofmt .govet .golint .gotest

cmd/runtimetest/main.go

@@ -1204,6 +1204,16 @@ func (c *complianceTester) validateMountLabel(spec *rspec.Spec) error {
 	}
 
 	for _, mount := range spec.Mounts {
+		isBind := false
+		for _, opt := range mount.Options {
+			if opt == "bind" || opt == "rbind" {
+				isBind = true
+				break
+			}
+		}
+		if !isBind {
+			continue
+		}
 		fileLabel, err := label.FileLabel(mount.Destination)
 		if err != nil {
 			return fmt.Errorf("Failed to get mountLabel of %v", mount.Destination)

validation/delete/delete.go

@@ -16,6 +16,8 @@ import (
 func main() {
 	t := tap.New()
 	t.Header(0)
+	defer t.AutoPlan()
+
 	bundleDir, err := util.PrepareBundle()
 	if err != nil {
 		util.Fatal(err)
@@ -72,7 +74,7 @@ func main() {
 
 		if c.effectCheck {
 			// waiting for the error of State, just in case the delete operation takes time
-			util.WaitingForStatus(testRuntime, util.LifecycleActionNone, time.Second*10, time.Second*1)
+			util.WaitingForStatus(testRuntime, util.LifecycleActionNone, time.Second*3, time.Second/2)
 			_, err = testRuntime.State()
 			// err == nil means the 'delete' operation does NOT take effect
 			util.SpecErrorOK(t, err == nil, specerror.NewError(specerror.DeleteNonStopHaveNoEffect, fmt.Errorf("attempting to `delete` a container that is not `stopped` MUST have no effect on the container"), rspecs.Version), err)
@@ -89,6 +91,4 @@ func main() {
 			}
 		}
 	}
-
-	t.AutoPlan()
 }

validation/kill/kill.go

@@ -70,6 +70,10 @@ func main() {
 				// KILL MUST be supported and KILL cannot be trapped
 				err = r.Kill("KILL")
 				util.WaitingForStatus(*r, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
+				if err != nil {
+					//Be sure to not leave the container around
+					r.Delete()
+				}
 				return err
 			},
 		}

validation/linux_seccomp/linux_seccomp.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	tap "github.com/mndrix/tap-go"
 	"github.com/opencontainers/runtime-tools/generate/seccomp"
 	"github.com/opencontainers/runtime-tools/validation/util"
 )
 
 func main() {
+	t := tap.New()
+	t.Header(0)
+	defer t.AutoPlan()
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		util.Fatal(err)
@@ -16,8 +20,10 @@ func main() {
 	}
 	g.SetDefaultSeccompAction("allow")
 	g.SetSyscallAction(syscallArgs)
-	err = util.RuntimeInsideValidate(g, nil, nil)
+	err = util.RuntimeInsideValidate(g, t, nil)
+	t.Ok(err == nil, "seccomp action is added correctly")
 	if err != nil {
-		util.Fatal(err)
+		t.Fail(err.Error())
 	}
+
 }

validation/misc_props/misc_props.go

@@ -45,9 +45,15 @@ func main() {
 		util.Fatal(err)
 	}
 	basicConfig.SetProcessArgs([]string{"true"})
-	annotationConfig := basicConfig
+	annotationConfig, err := util.GetDefaultGenerator()
+	if err != nil {
+		util.Fatal(err)
+	}
 	annotationConfig.AddAnnotation(fmt.Sprintf("org.%s", containerID), "")
-	invalidConfig := basicConfig
+	invalidConfig, err := util.GetDefaultGenerator()
+	if err != nil {
+		util.Fatal(err)
+	}
 	invalidConfig.SetVersion("invalid")
 
 	cases := []struct {

validation/pidfile/pidfile.go

@@ -7,6 +7,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strconv"
+	"time"
 
 	tap "github.com/mndrix/tap-go"
 	"github.com/opencontainers/runtime-tools/validation/util"
@@ -31,7 +32,7 @@ func main() {
 	g.SetProcessArgs([]string{"true"})
 	config := util.LifecycleConfig{
 		Config:  g,
-		Actions: util.LifecycleActionCreate | util.LifecycleActionDelete,
+		Actions: util.LifecycleActionCreate | util.LifecycleActionStart | util.LifecycleActionDelete,
 		PreCreate: func(r *util.Runtime) error {
 			r.SetID(uuid.NewV4().String())
 			r.PidFile = tempPidFile
@@ -55,6 +56,13 @@ func main() {
 			}
 			return nil
 		},
+		PreDelete: func(r *util.Runtime) error {
+			util.WaitingForStatus(*r, util.LifecycleStatusRunning, time.Second*10, time.Second*1)
+			err = r.Kill("KILL")
+			// wait before the container been deleted
+			util.WaitingForStatus(*r, util.LifecycleStatusStopped, time.Second*10, time.Second*1)
+			return err
+		},
 	}
 
 	err = util.RuntimeLifecycleValidate(config)

validation/process_capabilities_fail/process_capabilities_fail.go

@@ -20,7 +20,7 @@ func main() {
 	if err != nil {
 		util.Fatal(err)
 	}
-	g.AddProcessCapabilityBounding("CAP_TEST")
+	g.Config.Process.Capabilities.Bounding = append(g.Config.Process.Capabilities.Bounding, "CAP_TEST")
 	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err == nil {
 		util.Fatal(specerror.NewError(specerror.LinuxProcCapError, fmt.Errorf("Any value which cannot be mapped to a relevant kernel interface MUST cause an error"), rspecs.Version))

validation/state/state.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os/exec"
+	"time"
 
 	"github.com/mndrix/tap-go"
 	rspecs "github.com/opencontainers/runtime-spec/specs-go"
@@ -44,6 +45,8 @@ func main() {
 			},
 			PostCreate: func(r *util.Runtime) error {
 				_, err = r.State()
+				r.Kill("KILL")
+				util.WaitingForStatus(*r, util.LifecycleStatusStopped, time.Second*10, time.Second)
 				return err
 			},
 		}

validation/util/container.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"time"
 
 	rspecs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
@@ -197,6 +198,9 @@ func (r *Runtime) Delete() (err error) {
 // forceRemoveBundle is true, after the deletion attempt regardless of
 // whether it was successful or not.
 func (r *Runtime) Clean(removeBundle bool, forceRemoveBundle bool) error {
+	r.Kill("KILL")
+	WaitingForStatus(*r, LifecycleStatusStopped, time.Second*10, time.Second/10)
+
 	err := r.Delete()
 
 	if removeBundle && (err == nil || forceRemoveBundle) {

validation/util/test.go

@@ -350,7 +350,8 @@ func RuntimeLifecycleValidate(config LifecycleConfig) error {
 				if _, err := r.State(); err != nil {
 					return
 				}
-				err := WaitingForStatus(r, LifecycleStatusCreated|LifecycleStatusStopped, time.Second*10, time.Second*1)
+				r.Kill("KILL")
+				err := WaitingForStatus(r, LifecycleStatusStopped, time.Second*10, time.Second*1)
 				if err == nil {
 					r.Delete()
 				} else {