add tests when prestart/poststart/poststop hooks fail

validation/pidfile.go

@@ -24,7 +24,10 @@ func main() {
 	defer os.RemoveAll(tempDir)
 	tempPidFile := filepath.Join(tempDir, "pidfile")
 
+	g := util.GetDefaultGenerator()
+	g.SetProcessArgs([]string{"true"})
 	config := util.LifecycleConfig{
+		Config:  g,
 		Actions: util.LifecycleActionCreate | util.LifecycleActionDelete,
 		PreCreate: func(r *util.Runtime) error {
 			r.SetID(uuid.NewV4().String())
@@ -51,9 +54,7 @@ func main() {
 		},
 	}
 
-	g := util.GetDefaultGenerator()
-	g.SetProcessArgs([]string{"true"})
-	err = util.RuntimeLifecycleValidate(g, config)
+	err = util.RuntimeLifecycleValidate(config)
 	t.Ok(err == nil, "create with '--pid-file' option works")
 	if err != nil {
 		diagnostic := map[string]string{

validation/poststart.go

@@ -28,7 +28,7 @@ func main() {
 			g := util.GetDefaultGenerator()
 			output = filepath.Join(r.BundleDir, g.Spec().Root.Path, "output")
 			poststart := rspec.Hook{
-				Path: fmt.Sprintf("%s/%s/bin/sh", r.BundleDir, g.Spec().Root.Path),
+				Path: filepath.Join(r.BundleDir, g.Spec().Root.Path, "/bin/sh"),
 				Args: []string{
 					"sh", "-c", fmt.Sprintf("echo 'post-start called' >> %s", output),
 				},
@@ -73,7 +73,7 @@ func main() {
 		},
 	}
 
-	err := util.RuntimeLifecycleValidate(nil, config)
+	err := util.RuntimeLifecycleValidate(config)
 	if err != nil {
 		diagnostic := map[string]string{
 			"error": err.Error(),

validation/poststart_fail.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	tap "github.com/mndrix/tap-go"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/specerror"
+	"github.com/opencontainers/runtime-tools/validation/util"
+	uuid "github.com/satori/go.uuid"
+)
+
+func main() {
+	t := tap.New()
+	t.Header(0)
+
+	bundleDir, err := util.PrepareBundle()
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(bundleDir)
+
+	g := util.GetDefaultGenerator()
+	output := filepath.Join(bundleDir, g.Spec().Root.Path, "output")
+	poststart := rspec.Hook{
+		Path: filepath.Join(bundleDir, g.Spec().Root.Path, "/bin/false"),
+		Args: []string{"false"},
+	}
+	g.AddPostStartHook(poststart)
+	poststartOK := rspec.Hook{
+		Path: filepath.Join(bundleDir, g.Spec().Root.Path, "/bin/sh"),
+		Args: []string{
+			"sh", "-c", fmt.Sprintf("echo 'post-start called' >> %s", output),
+		},
+	}
+	g.AddPostStartHook(poststartOK)
+	g.SetProcessArgs([]string{"true"})
+	config := util.LifecycleConfig{
+		Config:    g,
+		BundleDir: bundleDir,
+		Actions:   util.LifecycleActionCreate | util.LifecycleActionStart | util.LifecycleActionDelete,
+		PreCreate: func(r *util.Runtime) error {
+			r.SetID(uuid.NewV4().String())
+			return nil
+		},
+		PreDelete: func(r *util.Runtime) error {
+			util.WaitingForStatus(*r, util.LifecycleStatusStopped, time.Second*10, time.Second)
+			return nil
+		},
+	}
+
+	runErr := util.RuntimeLifecycleValidate(config)
+	outputData, _ := ioutil.ReadFile(output)
+
+	// if runErr is not nil, it means the runtime generates an error
+	// if outputData is not equal to the expected content, it means there is something wrong with the remaining hooks and lifecycle
+	if runErr != nil || string(outputData) != "post-start called\n" {
+		err := specerror.NewError(specerror.PoststartHookFailGenWarn, fmt.Errorf("if any poststart hook fails, the runtime MUST log a warning, but the remaining hooks and lifecycle continue as if the hook had succeeded"), rspec.Version)
+		diagnostic := map[string]string{
+			"error": err.Error(),
+		}
+		t.YAML(diagnostic)
+	}
+
+	t.AutoPlan()
+}

validation/poststop.go

@@ -29,7 +29,7 @@ func main() {
 			g := util.GetDefaultGenerator()
 			output = filepath.Join(r.BundleDir, g.Spec().Root.Path, "output")
 			poststop := rspec.Hook{
-				Path: fmt.Sprintf("%s/%s/bin/sh", r.BundleDir, g.Spec().Root.Path),
+				Path: filepath.Join(r.BundleDir, g.Spec().Root.Path, "/bin/sh"),
 				Args: []string{
 					"sh", "-c", fmt.Sprintf("echo 'post-stop called' >> %s", output),
 				},
@@ -90,7 +90,7 @@ func main() {
 		},
 	}
 
-	err := util.RuntimeLifecycleValidate(nil, config)
+	err := util.RuntimeLifecycleValidate(config)
 	if err != nil {
 		diagnostic := map[string]string{
 			"error": err.Error(),

validation/poststop_fail.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	tap "github.com/mndrix/tap-go"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/specerror"
+	"github.com/opencontainers/runtime-tools/validation/util"
+	uuid "github.com/satori/go.uuid"
+)
+
+func main() {
+	t := tap.New()
+	t.Header(0)
+
+	bundleDir, err := util.PrepareBundle()
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(bundleDir)
+
+	g := util.GetDefaultGenerator()
+	output := filepath.Join(bundleDir, g.Spec().Root.Path, "output")
+	poststop := rspec.Hook{
+		Path: filepath.Join(bundleDir, g.Spec().Root.Path, "/bin/false"),
+		Args: []string{"false"},
+	}
+	g.AddPostStopHook(poststop)
+	poststopOK := rspec.Hook{
+		Path: filepath.Join(bundleDir, g.Spec().Root.Path, "/bin/sh"),
+		Args: []string{
+			"sh", "-c", fmt.Sprintf("echo 'post-stop called' >> %s", output),
+		},
+	}
+	g.AddPostStopHook(poststopOK)
+	g.SetProcessArgs([]string{"true"})
+
+	config := util.LifecycleConfig{
+		Config:    g,
+		BundleDir: bundleDir,
+		Actions:   util.LifecycleActionCreate | util.LifecycleActionStart | util.LifecycleActionDelete,
+		PreCreate: func(r *util.Runtime) error {
+			r.SetID(uuid.NewV4().String())
+			return nil
+		},
+		PreDelete: func(r *util.Runtime) error {
+			util.WaitingForStatus(*r, util.LifecycleStatusStopped, time.Second*10, time.Second)
+			return nil
+		},
+	}
+
+	runErr := util.RuntimeLifecycleValidate(config)
+	outputData, _ := ioutil.ReadFile(output)
+	// if runErr is not nil, it means the runtime generates an error
+	// if outputData is not equal to the expected content, it means there is something wrong with the remaining hooks and lifecycle
+	if runErr != nil || string(outputData) != "post-stop called\n" {
+		err := specerror.NewError(specerror.PoststopHookFailGenWarn, fmt.Errorf("if any poststop hook fails, the runtime MUST log a warning, but the remaining hooks and lifecycle continue as if the hook had succeeded"), rspec.Version)
+		diagnostic := map[string]string{
+			"error": err.Error(),
+		}
+		t.YAML(diagnostic)
+	}
+
+	t.AutoPlan()
+}

validation/prestart.go

@@ -27,7 +27,7 @@ func main() {
 			g := util.GetDefaultGenerator()
 			output = filepath.Join(r.BundleDir, g.Spec().Root.Path, "output")
 			prestart := rspec.Hook{
-				Path: fmt.Sprintf("%s/%s/bin/sh", r.BundleDir, g.Spec().Root.Path),
+				Path: filepath.Join(r.BundleDir, g.Spec().Root.Path, "/bin/sh"),
 				Args: []string{
 					"sh", "-c", fmt.Sprintf("echo 'pre-start called' >> %s", output),
 				},
@@ -72,7 +72,7 @@ func main() {
 		},
 	}
 
-	err := util.RuntimeLifecycleValidate(nil, config)
+	err := util.RuntimeLifecycleValidate(config)
 	if err != nil {
 		diagnostic := map[string]string{
 			"error": err.Error(),

validation/prestart_fail.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	tap "github.com/mndrix/tap-go"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/specerror"
+	"github.com/opencontainers/runtime-tools/validation/util"
+	uuid "github.com/satori/go.uuid"
+)
+
+func main() {
+	t := tap.New()
+	t.Header(0)
+
+	bundleDir, err := util.PrepareBundle()
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(bundleDir)
+
+	g := util.GetDefaultGenerator()
+	prestart := rspec.Hook{
+		Path: filepath.Join(bundleDir, g.Spec().Root.Path, "/bin/false"),
+		Args: []string{"false"},
+	}
+	g.AddPreStartHook(prestart)
+	g.SetProcessArgs([]string{"sh", "-c", fmt.Sprintf("touch %s", "/output")})
+	containerID := uuid.NewV4().String()
+
+	config := util.LifecycleConfig{
+		Config:    g,
+		BundleDir: bundleDir,
+		Actions:   util.LifecycleActionCreate | util.LifecycleActionStart,
+		PreCreate: func(r *util.Runtime) error {
+			r.SetID(containerID)
+			return nil
+		},
+	}
+
+	runErr := util.RuntimeLifecycleValidate(config)
+	_, outputErr := os.Stat(filepath.Join(bundleDir, g.Spec().Root.Path, "output"))
+
+	// query the state
+	r, _ := util.NewRuntime(util.RuntimeCommand, "")
+	r.SetID(containerID)
+	_, stateErr := r.State()
+	if stateErr != nil {
+		// In case a container is created, delete it
+		r.Delete()
+	}
+
+	// if runErr is nil, it means the runtime does not generate an error
+	// if outputErr is nil, it means the runtime calls the Process anyway
+	// if stateErr is nil, it means it does not continue lifecycle at step 9
+	if runErr == nil || outputErr == nil || stateErr == nil {
+		err := specerror.NewError(specerror.PrestartHookFailGenError, fmt.Errorf("if any prestart hook fails, the runtime MUST generate an error, stop the container, and continue the lifecycle at step 9"), rspec.Version)
+		diagnostic := map[string]string{
+			"error": err.Error(),
+		}
+		t.YAML(diagnostic)
+	}
+
+	t.AutoPlan()
+}

validation/state.go

@@ -32,6 +32,7 @@ func main() {
 
 	for _, c := range cases {
 		config := util.LifecycleConfig{
+			Config:  g,
 			Actions: c.action,
 			PreCreate: func(r *util.Runtime) error {
 				r.SetID(c.id)
@@ -42,7 +43,7 @@ func main() {
 				return err
 			},
 		}
-		err := util.RuntimeLifecycleValidate(g, config)
+		err := util.RuntimeLifecycleValidate(config)
 		t.Ok((err == nil) == c.errExpected, c.err.(*specerror.Error).Err.Err.Error())
 		diagnostic := map[string]string{
 			"reference": c.err.(*specerror.Error).Err.Reference,

validation/util/test.go

@@ -58,9 +58,13 @@ var lifecycleStatusMap = map[string]LifecycleStatus{
 }
 
 // LifecycleConfig includes
-// 1. Actions to define the default running lifecycles.
-// 2. Four phases for user to add his/her own operations.
+// 1. Config to set the 'config.json'
+// 2. BundleDir to set the bundle directory
+// 3. Actions to define the default running lifecycles
+// 4. Four phases for user to add his/her own operations
 type LifecycleConfig struct {
+	Config     *generate.Generator
+	BundleDir  string
 	Actions    LifecycleAction
 	PreCreate  func(runtime *Runtime) error
 	PostCreate func(runtime *Runtime) error
@@ -258,19 +262,27 @@ func RuntimeOutsideValidate(g *generate.Generator, f AfterFunc) error {
 }
 
 // RuntimeLifecycleValidate validates runtime lifecycle.
-func RuntimeLifecycleValidate(g *generate.Generator, config LifecycleConfig) error {
-	bundleDir, err := PrepareBundle()
-	if err != nil {
-		return err
+func RuntimeLifecycleValidate(config LifecycleConfig) error {
+	var bundleDir string
+	var err error
+
+	if config.BundleDir == "" {
+		bundleDir, err = PrepareBundle()
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(bundleDir)
+	} else {
+		bundleDir = config.BundleDir
 	}
-	defer os.RemoveAll(bundleDir)
+
 	r, err := NewRuntime(RuntimeCommand, bundleDir)
 	if err != nil {
 		return err
 	}
 
-	if g != nil {
-		if err := r.SetConfig(g); err != nil {
+	if config.Config != nil {
+		if err := r.SetConfig(config.Config); err != nil {
 			return err
 		}
 	}