Merge pull request #1013 from liubin/feature/1012-dump-guest-memroy-on-panic

Dump guest memory when kernel panic for QEMU

src/runtime/cli/config/configuration-qemu.toml.in

@@ -319,6 +319,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
 # Default 0-sized value means unlimited rate.
 #tx_rate_limiter_max_rate = 0
 
+# Set where to save the guest memory dump file.
+# If set, when GUEST_PANICKED event occurred,
+# guest memeory will be dumped to host filesystem under guest_memory_dump_path,
+# This directory will be created automatically if it does not exist.
+#
+# The dumped file(also called vmcore) can be processed with crash or gdb.
+#
+# WARNING:
+#   Dump guest’s memory can take very long depending on the amount of guest memory
+#   and use much disk space.
+#guest_memory_dump_path="/var/crash/kata"
+
+# If enable paging.
+# Basically, if you want to use "gdb" rather than "crash",
+# or need the guest-virtual addresses in the ELF vmcore,
+# then you should enable paging.
+#
+# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details
+#guest_memory_dump_paging=false
+
 [factory]
 # VM templating support. Once enabled, new VMs are created from template
 # using vm cloning. They will share the same initial kernel, initramfs and

src/runtime/cli/kata-env.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -292,7 +293,7 @@ func getNetmonInfo(config oci.RuntimeConfig) NetmonInfo {
 }
 
 func getCommandVersion(cmd string) (string, error) {
-	return katautils.RunCommand([]string{cmd, "--version"})
+	return utils.RunCommand([]string{cmd, "--version"})
 }
 
 func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {

src/runtime/cli/main_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/dlespiau/covertool/pkg/cover"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -273,7 +274,7 @@ func createOCIConfig(bundleDir string) error {
 		return errors.New("Cannot find command to generate OCI config file")
 	}
 
-	_, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
 	if err != nil {
 		return err
 	}
@@ -378,7 +379,7 @@ func makeOCIBundle(bundleDir string) error {
 		}
 	}
 
-	output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true)
+	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
 	if err != nil {
 		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
 	}

src/runtime/cli/utils_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -176,13 +177,13 @@ VERSION_ID="%s"
 }
 
 func TestUtilsRunCommand(t *testing.T) {
-	output, err := katautils.RunCommand([]string{"true"})
+	output, err := utils.RunCommand([]string{"true"})
 	assert.NoError(t, err)
 	assert.Equal(t, "", output)
 }
 
 func TestUtilsRunCommandCaptureStdout(t *testing.T) {
-	output, err := katautils.RunCommand([]string{"echo", "hello"})
+	output, err := utils.RunCommand([]string{"echo", "hello"})
 	assert.NoError(t, err)
 	assert.Equal(t, "hello", output)
 }
@@ -190,7 +191,7 @@ func TestUtilsRunCommandCaptureStdout(t *testing.T) {
 func TestUtilsRunCommandIgnoreStderr(t *testing.T) {
 	args := []string{"/bin/sh", "-c", "echo foo >&2;exit 0"}
 
-	output, err := katautils.RunCommand(args)
+	output, err := utils.RunCommand(args)
 	assert.NoError(t, err)
 	assert.Equal(t, "", output)
 }
@@ -213,7 +214,7 @@ func TestUtilsRunCommandInvalidCmds(t *testing.T) {
 	}
 
 	for _, args := range invalidCommands {
-		output, err := katautils.RunCommand(args)
+		output, err := utils.RunCommand(args)
 		assert.Error(t, err)
 		assert.Equal(t, "", output)
 	}

src/runtime/containerd-shim-v2/utils_test.go

@@ -21,6 +21,7 @@ import (
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -150,7 +151,7 @@ func createOCIConfig(bundleDir string) error {
 		return errors.New("Cannot find command to generate OCI config file")
 	}
 
-	_, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
 	if err != nil {
 		return err
 	}
@@ -278,7 +279,7 @@ func makeOCIBundle(bundleDir string) error {
 		}
 	}
 
-	output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true)
+	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
 	if err != nil {
 		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
 	}

src/runtime/go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/gogo/protobuf v1.3.1
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/intel/govmm v0.0.0-20200825065022-6042f6033126
+	github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864
 	github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v1.0.0-rc9.0.20200102164712-2b52db75279c

src/runtime/go.sum

@@ -186,8 +186,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/intel/govmm v0.0.0-20200825065022-6042f6033126 h1:yltaUdR0Vitnn/FEfy+JWbJ+oGhMAPP/3S7ja9S5yso=
-github.com/intel/govmm v0.0.0-20200825065022-6042f6033126/go.mod h1:QKGWoQtjvkvFtzP6ybiM3lxUHqf83Sv3oLqyELUKH4g=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -198,7 +196,8 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0b
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kata-containers/kata-containers v0.0.0-20201013034856-c88820454d08 h1:yk9fzLKb9RmV9xuT5mkJw4owk/K0rX5cusm2ukEEDro=
+github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864 h1:ETwjbdr9aU/J90P5D/HAxRW8M4r0HQSPmuBDIaNr9EM=
+github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=

src/runtime/pkg/katautils/config.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
@@ -125,6 +125,8 @@ type hypervisor struct {
 	RxRateLimiterMaxRate    uint64   `toml:"rx_rate_limiter_max_rate"`
 	TxRateLimiterMaxRate    uint64   `toml:"tx_rate_limiter_max_rate"`
 	EnableAnnotations       []string `toml:"enable_annotations"`
+	GuestMemoryDumpPath     string   `toml:"guest_memory_dump_path"`
+	GuestMemoryDumpPaging   bool     `toml:"guest_memory_dump_paging"`
 }
 
 type runtime struct {
@@ -688,6 +690,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		RxRateLimiterMaxRate:    rxRateLimiterMaxRate,
 		TxRateLimiterMaxRate:    txRateLimiterMaxRate,
 		EnableAnnotations:       h.EnableAnnotations,
+		GuestMemoryDumpPath:     h.GuestMemoryDumpPath,
+		GuestMemoryDumpPaging:   h.GuestMemoryDumpPaging,
 	}, nil
 }
 
@@ -844,7 +848,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		DisableVhostNet:         true,
 		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
 		SGXEPCSize:              defaultSGXEPCSize,
-		EnableAnnotations:     h.EnableAnnotations,
+		EnableAnnotations:       h.EnableAnnotations,
 	}, nil
 }
 

src/runtime/pkg/katautils/container_engine.go

@@ -7,6 +7,8 @@ package katautils
 
 import (
 	"os/exec"
+
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 )
 
 type CtrEngine struct {
@@ -19,7 +21,7 @@ var (
 
 func (e *CtrEngine) Init(name string) (string, error) {
 	var out string
-	out, err := RunCommandFull([]string{name, "version"}, true)
+	out, err := utils.RunCommandFull([]string{name, "version"}, true)
 	if err != nil {
 		return out, err
 	}
@@ -30,19 +32,19 @@ func (e *CtrEngine) Init(name string) (string, error) {
 
 func (e *CtrEngine) Inspect(image string) (string, error) {
 	// Only hit the network if the image doesn't exist locally
-	return RunCommand([]string{e.Name, "inspect", "--type=image", image})
+	return utils.RunCommand([]string{e.Name, "inspect", "--type=image", image})
 }
 
 func (e *CtrEngine) Pull(image string) (string, error) {
-	return RunCommand([]string{e.Name, "pull", image})
+	return utils.RunCommand([]string{e.Name, "pull", image})
 }
 
 func (e *CtrEngine) Create(image string) (string, error) {
-	return RunCommand([]string{e.Name, "create", image})
+	return utils.RunCommand([]string{e.Name, "create", image})
 }
 
 func (e *CtrEngine) Rm(ctrID string) (string, error) {
-	return RunCommand([]string{e.Name, "rm", ctrID})
+	return utils.RunCommand([]string{e.Name, "rm", ctrID})
 }
 
 func (e *CtrEngine) GetRootfs(ctrID string, dir string) error {

src/runtime/pkg/katautils/create_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
@@ -87,7 +88,7 @@ func makeOCIBundle(bundleDir string) error {
 		}
 	}
 
-	output, err := RunCommandFull([]string{"cp", "-a", from, to}, true)
+	output, err := utils.RunCommandFull([]string{"cp", "-a", from, to}, true)
 	if err != nil {
 		return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output)
 	}

src/runtime/pkg/katautils/utils.go

@@ -8,13 +8,12 @@ package katautils
 
 import (
 	"fmt"
-	"golang.org/x/sys/unix"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
 	"syscall"
+
+	"golang.org/x/sys/unix"
 )
 
 // FileExists test is a file exiting or not
@@ -110,27 +109,3 @@ func GetFileContents(file string) (string, error) {
 
 	return string(bytes), nil
 }
-
-// RunCommandFull returns the commands space-trimmed standard output and
-// error on success. Note that if the command fails, the requested output will
-// still be returned, along with an error.
-func RunCommandFull(args []string, includeStderr bool) (string, error) {
-	cmd := exec.Command(args[0], args[1:]...)
-	var err error
-	var bytes []byte
-
-	if includeStderr {
-		bytes, err = cmd.CombinedOutput()
-	} else {
-		bytes, err = cmd.Output()
-	}
-
-	trimmed := strings.TrimSpace(string(bytes))
-
-	return trimmed, err
-}
-
-// RunCommand returns the commands space-trimmed standard output on success
-func RunCommand(args []string) (string, error) {
-	return RunCommandFull(args, false)
-}

src/runtime/pkg/katautils/utils_test.go

@@ -18,6 +18,7 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	"github.com/stretchr/testify/assert"
 )
@@ -90,7 +91,7 @@ func createOCIConfig(bundleDir string) error {
 		return errors.New("Cannot find command to generate OCI config file")
 	}
 
-	_, err := RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
+	_, err := utils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir})
 	if err != nil {
 		return err
 	}

src/runtime/pkg/utils/utils.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020 Ant Financial
+// Copyright (c) 2020 Ant Group
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -6,7 +6,11 @@
 package utils
 
 import (
+	"fmt"
 	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 )
 
@@ -31,3 +35,48 @@ func GzipAccepted(header http.Header) bool {
 func String2Pointer(s string) *string {
 	return &s
 }
+
+// RunCommandFull returns the commands space-trimmed standard output and
+// error on success. Note that if the command fails, the requested output will
+// still be returned, along with an error.
+func RunCommandFull(args []string, includeStderr bool) (string, error) {
+	cmd := exec.Command(args[0], args[1:]...)
+	var err error
+	var bytes []byte
+
+	if includeStderr {
+		bytes, err = cmd.CombinedOutput()
+	} else {
+		bytes, err = cmd.Output()
+	}
+
+	trimmed := strings.TrimSpace(string(bytes))
+
+	return trimmed, err
+}
+
+// RunCommand returns the commands space-trimmed standard output on success
+func RunCommand(args []string) (string, error) {
+	return RunCommandFull(args, false)
+}
+
+// EnsureDir check if a directory exist, if not then create it
+func EnsureDir(path string, mode os.FileMode) error {
+	if !filepath.IsAbs(path) {
+		return fmt.Errorf("Not an absolute path: %s", path)
+	}
+
+	if fi, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			if err = os.MkdirAll(path, mode); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	} else if !fi.IsDir() {
+		return fmt.Errorf("Not a directory: %s", path)
+	}
+
+	return nil
+}

src/runtime/pkg/utils/utils_test.go

@@ -6,7 +6,10 @@
 package utils
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net/http"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -45,3 +48,71 @@ func TestGzipAccepted(t *testing.T) {
 		assert.Equal(tc.result, b)
 	}
 }
+
+func TestEnsureDir(t *testing.T) {
+	const testMode = 0755
+	tmpdir, err := ioutil.TempDir("", "TestEnsureDir")
+	assert := assert.New(t)
+
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir)
+
+	testCases := []struct {
+		before func()
+		path   string
+		err    bool
+		msg    string
+	}{
+		{
+			before: nil,
+			path:   "a/b/c",
+			err:    true,
+			msg:    "Not an absolute path",
+		},
+		{
+			before: nil,
+			path:   fmt.Sprintf("%s/abc/def", tmpdir),
+			err:    false,
+			msg:    "",
+		},
+		{
+			before: nil,
+			path:   fmt.Sprintf("%s/abc", tmpdir),
+			err:    false,
+			msg:    "",
+		},
+		{
+			before: func() {
+				err := os.MkdirAll(fmt.Sprintf("%s/abc/def", tmpdir), testMode)
+				assert.NoError(err)
+			},
+			path: fmt.Sprintf("%s/abc/def", tmpdir),
+			err:  false,
+			msg:  "",
+		},
+		{
+			before: func() {
+				// create a regular file
+				err := os.MkdirAll(fmt.Sprintf("%s/abc", tmpdir), testMode)
+				assert.NoError(err)
+				_, err = os.Create(fmt.Sprintf("%s/abc/file.txt", tmpdir))
+				assert.NoError(err)
+			},
+			path: fmt.Sprintf("%s/abc/file.txt", tmpdir),
+			err:  true,
+			msg:  "Not a directory",
+		},
+	}
+
+	for _, tc := range testCases {
+		if tc.before != nil {
+			tc.before()
+		}
+		err := EnsureDir(tc.path, testMode)
+		if tc.err {
+			assert.Contains(err.Error(), tc.msg, "error msg should contains: %s, but got %s", tc.msg, err.Error())
+		} else {
+			assert.Equal(err, nil, "failed for path: %s, except no error, but got %+v", tc.path, err)
+		}
+	}
+}

src/runtime/vendor/github.com/intel/govmm/CONTRIBUTORS.md

@@ -1,22 +0,0 @@
-This file is a partial list of contributors to the Virtual Machine
-Manager for Go project. To see the full list of contributors, see the
-revision history in source control.
-
-Contributors who wish to be recognized in this file should add
-themselves (or their employer, as appropriate).
-
-- afrosi@de.ibm.com
-- archana.m.shinde@intel.com
-- caoruidong@huawei.com
-- clare.chenhui@huawei.com
-- eric.ernst@intel.com
-- james.o.hunt@intel.com
-- jose.carlos.venegas.munoz@intel.com
-- julio.montes@intel.com
-- manohar.r.castelino@intel.com
-- mark.d.ryan@intel.com
-- robert.bradford@intel.com
-- sameo@linux.intel.com
-- sebastien.boeuf@intel.com
-- teawater@hyper.sh
-- xinda.zhao@intel.com

src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go

@@ -1084,6 +1084,8 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string {
 		deviceParams = append(deviceParams, fmt.Sprintf(",share-rw=on"))
 	}
 
+	deviceParams = append(deviceParams, fmt.Sprintf(",serial=%s", blkdev.ID))
+
 	blkParams = append(blkParams, fmt.Sprintf("id=%s", blkdev.ID))
 	blkParams = append(blkParams, fmt.Sprintf(",file=%s", blkdev.File))
 	blkParams = append(blkParams, fmt.Sprintf(",aio=%s", blkdev.AIO))
@@ -1118,6 +1120,24 @@ func (blkdev BlockDevice) deviceName(config *Config) string {
 	return string(blkdev.Driver)
 }
 
+// PVPanicDevice represents a qemu pvpanic device.
+type PVPanicDevice struct {
+	NoShutdown bool
+}
+
+// Valid always returns true for pvpanic device
+func (dev PVPanicDevice) Valid() bool {
+	return true
+}
+
+// QemuParams returns the qemu parameters built out of this serial device.
+func (dev PVPanicDevice) QemuParams(config *Config) []string {
+	if dev.NoShutdown {
+		return []string{"-device", "pvpanic", "-no-shutdown"}
+	}
+	return []string{"-device", "pvpanic"}
+}
+
 // VhostUserDevice represents a qemu vhost-user device meant to be passed
 // in to the guest
 type VhostUserDevice struct {
@@ -2103,6 +2123,56 @@ type Kernel struct {
 	Params string
 }
 
+// FwCfg allows QEMU to pass entries to the guest
+// File and Str are mutually exclusive
+type FwCfg struct {
+	Name string
+	File string
+	Str  string
+}
+
+// Valid returns true if the FwCfg structure is valid and complete.
+func (fwcfg FwCfg) Valid() bool {
+	if fwcfg.Name == "" {
+		return false
+	}
+
+	if fwcfg.File != "" && fwcfg.Str != "" {
+		return false
+	}
+
+	if fwcfg.File == "" && fwcfg.Str == "" {
+		return false
+	}
+
+	return true
+}
+
+// QemuParams returns the qemu parameters built out of the FwCfg object
+func (fwcfg FwCfg) QemuParams(config *Config) []string {
+	var fwcfgParams []string
+	var qemuParams []string
+
+	for _, f := range config.FwCfg {
+		if f.Name != "" {
+			fwcfgParams = append(fwcfgParams, fmt.Sprintf("name=%s", f.Name))
+
+			if f.File != "" {
+				fwcfgParams = append(fwcfgParams, fmt.Sprintf(",file=%s", f.File))
+			}
+
+			if f.Str != "" {
+				fwcfgParams = append(fwcfgParams, fmt.Sprintf(",string=%s", f.Str))
+			}
+		}
+
+		qemuParams = append(qemuParams, "-fw_cfg")
+		qemuParams = append(qemuParams, strings.Join(fwcfgParams, ""))
+	}
+
+	return qemuParams
+}
+
 // Knobs regroups a set of qemu boolean settings
 type Knobs struct {
 	// NoUserConfig prevents qemu from loading user config files.
@@ -2236,6 +2306,9 @@ type Config struct {
 	// fds is a list of open file descriptors to be passed to the spawned qemu process
 	fds []*os.File
 
+	// FwCfg is the -fw_cfg parameter
+	FwCfg []FwCfg
+
 	IOThreads []IOThread
 
 	// PidFile is the -pidfile parameter
@@ -2568,6 +2641,21 @@ func (config *Config) appendLogFile() {
 	}
 }
 
+func (config *Config) appendFwCfg(logger QMPLog) {
+	if logger == nil {
+		logger = qmpNullLogger{}
+	}
+
+	for _, f := range config.FwCfg {
+		if !f.Valid() {
+			logger.Errorf("fw_cfg is not valid: %+v", config.FwCfg)
+			continue
+		}
+
+		config.qemuParams = append(config.qemuParams, f.QemuParams(config)...)
+	}
+}
+
 // LaunchQemu can be used to launch a new qemu instance.
 //
 // The Config parameter contains a set of qemu parameters and settings.
@@ -2595,6 +2683,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) {
 	config.appendIncoming()
 	config.appendPidFile()
 	config.appendLogFile()
+	config.appendFwCfg(logger)
 
 	if err := config.appendCPUs(); err != nil {
 		return "", err

src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go

@@ -268,7 +268,7 @@ func (q *QMP) readLoop(fromVMCh chan<- []byte) {
 	for scanner.Scan() {
 		line := scanner.Bytes()
 		if q.cfg.Logger.V(1) {
-			q.cfg.Logger.Infof("%s", string(line))
+			q.cfg.Logger.Infof("read from QMP: %s", string(line))
 		}
 
 		// Since []byte channel type transfer slice info(include slice underlying array pointer, len, cap)
@@ -1639,3 +1639,29 @@ func (q *QMP) ExecQomSet(ctx context.Context, path, property string, value uint6
 
 	return q.executeCommand(ctx, "qom-set", args, nil)
 }
+
+// ExecQomGet qom-get path property
+func (q *QMP) ExecQomGet(ctx context.Context, path, property string) (interface{}, error) {
+	args := map[string]interface{}{
+		"path":     path,
+		"property": property,
+	}
+
+	response, err := q.executeCommandWithResponse(ctx, "qom-get", args, nil, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return response, nil
+}
+
+// ExecuteDumpGuestMemory dump guest memory to host
+func (q *QMP) ExecuteDumpGuestMemory(ctx context.Context, protocol string, paging bool, format string) error {
+	args := map[string]interface{}{
+		"protocol": protocol,
+		"paging":   paging,
+		"format":   format,
+	}
+
+	return q.executeCommand(ctx, "dump-guest-memory", args, nil)
+}

src/runtime/vendor/github.com/sirupsen/logrus/go.mod

@@ -1,5 +1,7 @@
 module github.com/sirupsen/logrus
 
+go 1.15
+
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.1

src/runtime/vendor/modules.txt

@@ -222,9 +222,9 @@ github.com/hashicorp/errwrap
 # github.com/hashicorp/go-multierror v1.0.0
 ## explicit
 github.com/hashicorp/go-multierror
-# github.com/intel/govmm v0.0.0-20200825065022-6042f6033126
+# github.com/kata-containers/govmm v0.0.0-20201020052039-99f43ec18864
 ## explicit
-github.com/intel/govmm/qemu
+github.com/kata-containers/govmm/qemu
 # github.com/konsorten/go-windows-terminal-sequences v1.0.1
 github.com/konsorten/go-windows-terminal-sequences
 # github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329

src/runtime/virtcontainers/hypervisor.go

@@ -440,6 +440,13 @@ type HypervisorConfig struct {
 
 	// Enable annotations by name
 	EnableAnnotations []string
+
+	// GuestCoredumpPath is the path in host for saving guest memory dump
+	GuestMemoryDumpPath string
+
+	// GuestMemoryDumpPaging is used to indicate if enable paging
+	// for QEMU dump-guest-memory command
+	GuestMemoryDumpPaging bool
 }
 
 // vcpu mapping from vcpu number to thread number
@@ -607,6 +614,10 @@ func (conf *HypervisorConfig) HypervisorAssetPath() (string, error) {
 	return conf.assetPath(types.HypervisorAsset)
 }
 
+func (conf *HypervisorConfig) IfPVPanicEnabled() bool {
+	return conf.GuestMemoryDumpPath != ""
+}
+
 // HypervisorCtlAssetPath returns the VM hypervisor ctl path
 func (conf *HypervisorConfig) HypervisorCtlAssetPath() (string, error) {
 	return conf.assetPath(types.HypervisorCtlAsset)

src/runtime/virtcontainers/qemu.go

@@ -24,13 +24,14 @@ import (
 	"time"
 	"unsafe"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
+	pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
@@ -99,6 +100,9 @@ type qemu struct {
 	stopped bool
 
 	store persistapi.PersistDriver
+
+	// if in memory dump progress
+	memoryDumpFlag sync.Mutex
 }
 
 const (
@@ -106,6 +110,9 @@ const (
 	qmpSocket     = "qmp.sock"
 	vhostFSSocket = "vhost-fs.sock"
 
+	// memory dump format will be set to elf
+	memoryDumpFormat = "elf"
+
 	qmpCapErrMsg  = "Failed to negoatiate QMP capabilities"
 	qmpExecCatCmd = "exec:cat"
 
@@ -408,13 +415,17 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
 		}
 	}
 
+	if q.config.IfPVPanicEnabled() {
+		// there should have no errors for pvpanic device
+		devices, _ = q.arch.appendPVPanicDevice(devices)
+	}
+
 	var ioThread *govmmQemu.IOThread
 	if q.config.BlockDeviceDriver == config.VirtioSCSI {
 		return q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
 	}
 
 	return devices, ioThread, nil
-
 }
 
 func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) govmmQemu.Incoming {
@@ -1027,7 +1038,13 @@ func (q *qemu) qmpSetup() error {
 		return nil
 	}
 
-	cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()}
+	events := make(chan govmmQemu.QMPEvent)
+	go q.loopQMPEvent(events)
+
+	cfg := govmmQemu.QMPConfig{
+		Logger:  newQMPLogger(),
+		EventCh: events,
+	}
 
 	// Auto-closed by QMPStart().
 	disconnectCh := make(chan struct{})
@@ -1050,6 +1067,136 @@ func (q *qemu) qmpSetup() error {
 	return nil
 }
 
+func (q *qemu) loopQMPEvent(event chan govmmQemu.QMPEvent) {
+	for {
+		select {
+		case e, open := <-event:
+			if !open {
+				q.Logger().Infof("QMP event channel closed")
+				return
+			}
+			q.Logger().WithField("event", e).Debug("got QMP event")
+			if e.Name == "GUEST_PANICKED" {
+				go q.handleGuestPanic()
+			}
+		}
+	}
+}
+
+func (q *qemu) handleGuestPanic() {
+	if err := q.dumpGuestMemory(q.config.GuestMemoryDumpPath); err != nil {
+		q.Logger().WithError(err).Error("failed to dump guest memory")
+	}
+
+	// TODO: how to notify the upper level sandbox to handle the error
+	// to do a fast fail(shutdown or others).
+	// tracked by https://github.com/kata-containers/kata-containers/issues/1026
+}
+
+// canDumpGuestMemory check if can do a guest memory dump operation.
+// for now it only ensure there must be double of VM size for free disk spaces
+func (q *qemu) canDumpGuestMemory(dumpSavePath string) error {
+	fs := unix.Statfs_t{}
+	if err := unix.Statfs(dumpSavePath, &fs); err != nil {
+		q.Logger().WithError(err).WithField("dumpSavePath", dumpSavePath).Error("failed to call Statfs")
+		return nil
+	}
+	availSpaceInBytes := fs.Bavail * uint64(fs.Bsize)
+	q.Logger().WithFields(
+		logrus.Fields{
+			"dumpSavePath":      dumpSavePath,
+			"availSpaceInBytes": availSpaceInBytes,
+		}).Info("get avail space")
+
+	// get guest memory size
+	guestMemorySizeInBytes := (uint64(q.config.MemorySize) + uint64(q.state.HotpluggedMemory)) << utils.MibToBytesShift
+	q.Logger().WithField("guestMemorySizeInBytes", guestMemorySizeInBytes).Info("get guest memory size")
+
+	// default we want ensure there are at least double of VM memory size free spaces available,
+	// this may complete one dump operation for one sandbox
+	exceptMemorySize := guestMemorySizeInBytes * 2
+	if availSpaceInBytes >= exceptMemorySize {
+		return nil
+	} else {
+		return fmt.Errorf("there are not enough free space to store memory dump file. Except %d bytes, but only %d bytes available", exceptMemorySize, availSpaceInBytes)
+	}
+}
+
+// dumpSandboxMetaInfo save meta information for debug purpose, includes:
+// hypervisor verison, sandbox/container state, hypervisor config
+func (q *qemu) dumpSandboxMetaInfo(dumpSavePath string) {
+	dumpStatePath := filepath.Join(dumpSavePath, "state")
+
+	// copy state from /run/vc/sbs to memory dump directory
+	statePath := filepath.Join(q.store.RunStoragePath(), q.id)
+	command := []string{"/bin/cp", "-ar", statePath, dumpStatePath}
+	q.Logger().WithField("command", command).Info("try to save sandbox state")
+	if output, err := pkgUtils.RunCommandFull(command, true); err != nil {
+		q.Logger().WithError(err).WithField("output", output).Error("failed to save state")
+	}
+	// save hypervisor meta information
+	fileName := filepath.Join(dumpSavePath, "hypervisor.conf")
+	data, _ := json.MarshalIndent(q.config, "", " ")
+	if err := ioutil.WriteFile(fileName, data, defaultFilePerms); err != nil {
+		q.Logger().WithError(err).WithField("hypervisor.conf", data).Error("write to hypervisor.conf file failed")
+	}
+
+	// save hypervisor version
+	hyperVisorVersion, err := pkgUtils.RunCommand([]string{q.config.HypervisorPath, "--version"})
+	if err != nil {
+		q.Logger().WithError(err).WithField("HypervisorPath", data).Error("failed to get hypervisor version")
+	}
+
+	fileName = filepath.Join(dumpSavePath, "hypervisor.version")
+	if err := ioutil.WriteFile(fileName, []byte(hyperVisorVersion), defaultFilePerms); err != nil {
+		q.Logger().WithError(err).WithField("hypervisor.version", data).Error("write to hypervisor.version file failed")
+	}
+}
+
+func (q *qemu) dumpGuestMemory(dumpSavePath string) error {
+	if dumpSavePath == "" {
+		return nil
+	}
+
+	q.memoryDumpFlag.Lock()
+	defer q.memoryDumpFlag.Unlock()
+
+	q.Logger().WithField("dumpSavePath", dumpSavePath).Info("try to dump guest memory")
+
+	dumpSavePath = filepath.Join(dumpSavePath, q.id)
+	dumpStatePath := filepath.Join(dumpSavePath, "state")
+	if err := pkgUtils.EnsureDir(dumpStatePath, DirMode); err != nil {
+		return err
+	}
+
+	// save meta information for sandbox
+	q.dumpSandboxMetaInfo(dumpSavePath)
+	q.Logger().Info("dump sandbox meta information completed")
+
+	// check device free space and estimated dump size
+	if err := q.canDumpGuestMemory(dumpSavePath); err != nil {
+		q.Logger().Warnf("can't dump guest memory: %s", err.Error())
+		return err
+	}
+
+	// dump guest memory
+	protocol := fmt.Sprintf("file:%s/vmcore-%s.%s", dumpSavePath, time.Now().Format("20060102150405.999"), memoryDumpFormat)
+	q.Logger().Infof("try to dump guest memory to %s", protocol)
+
+	if err := q.qmpSetup(); err != nil {
+		q.Logger().WithError(err).Error("setup manage QMP failed")
+		return err
+	}
+
+	if err := q.qmpMonitorCh.qmp.ExecuteDumpGuestMemory(q.qmpMonitorCh.ctx, protocol, q.config.GuestMemoryDumpPaging, memoryDumpFormat); err != nil {
+		q.Logger().WithError(err).Error("dump guest memory failed")
+		return err
+	}
+
+	q.Logger().Info("dump guest memory completed")
+	return nil
+}
+
 func (q *qemu) qmpShutdown() {
 	q.qmpMonitorCh.Lock()
 	defer q.qmpMonitorCh.Unlock()
@@ -2250,6 +2397,9 @@ func (q *qemu) load(s persistapi.HypervisorState) {
 }
 
 func (q *qemu) check() error {
+	q.memoryDumpFlag.Lock()
+	defer q.memoryDumpFlag.Unlock()
+
 	err := q.qmpSetup()
 	if err != nil {
 		return err

src/runtime/virtcontainers/qemu_amd64.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 )
 
 type qemuAmd64 struct {

src/runtime/virtcontainers/qemu_amd64_test.go

@@ -11,7 +11,7 @@ import (
 	"os"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/stretchr/testify/assert"
 )

src/runtime/virtcontainers/qemu_arch_base.go

@@ -14,7 +14,7 @@ import (
 	"strconv"
 	"strings"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
@@ -133,6 +133,9 @@ type qemuArch interface {
 
 	// append vIOMMU device
 	appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error)
+
+	// append pvpanic device
+	appendPVPanicDevice(devices []govmmQemu.Device) ([]govmmQemu.Device, error)
 }
 
 type qemuArchBase struct {
@@ -789,3 +792,9 @@ func (q *qemuArchBase) appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Devi
 		return devices, fmt.Errorf("Machine Type %s does not support vIOMMU", q.qemuMachine.Type)
 	}
 }
+
+// appendPVPanicDevice appends a pvpanic device
+func (q *qemuArchBase) appendPVPanicDevice(devices []govmmQemu.Device) ([]govmmQemu.Device, error) {
+	devices = append(devices, govmmQemu.PVPanicDevice{NoShutdown: true})
+	return devices, nil
+}

src/runtime/virtcontainers/qemu_arch_base_test.go

@@ -13,7 +13,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"

src/runtime/virtcontainers/qemu_arm64.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"time"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 )
 

src/runtime/virtcontainers/qemu_arm64_test.go

@@ -11,7 +11,7 @@ import (
 	"os"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/virtcontainers/qemu_ppc64le.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"time"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/sirupsen/logrus"
 )

src/runtime/virtcontainers/qemu_ppc64le_test.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/virtcontainers/qemu_s390x.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"time"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 )

src/runtime/virtcontainers/qemu_s390x_test.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/stretchr/testify/assert"
 )

src/runtime/virtcontainers/qemu_test.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"