diff --git a/Gopkg.lock b/Gopkg.lock index 80dcc8cb3..21ad8808c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -89,7 +89,6 @@ revision = "c4b9ac5c7601384c965b9646fc515884e091ebb9" [[projects]] - branch = "master" digest = "1:da4daad2ec1737eec4ebeeed7afedb631711f96bbac0c361a17a4d0369d00c6d" name = "github.com/containerd/console" packages = ["."] @@ -707,6 +706,7 @@ "github.com/BurntSushi/toml", "github.com/blang/semver", "github.com/containerd/cgroups", + "github.com/containerd/console", "github.com/containerd/containerd/api/events", "github.com/containerd/containerd/api/types", "github.com/containerd/containerd/api/types/task", diff --git a/Gopkg.toml b/Gopkg.toml index 076fc520b..d242baa2a 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -78,6 +78,10 @@ branch = "master" name = "github.com/hashicorp/yamux" +[[constraint]] + revision = "0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f" + name = "github.com/containerd/console" + [prune] non-go = true go-tests = true diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go index 5c69b8ae3..a734a7ab9 100644 --- a/virtcontainers/acrn.go +++ b/virtcontainers/acrn.go @@ -348,7 +348,7 @@ func (a *Acrn) createDummyVirtioBlkDev(devices []Device) ([]Device, error) { } // createSandbox is the Hypervisor sandbox creation. -func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { +func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error { // Save the tracing context a.ctx = ctx diff --git a/virtcontainers/acrn_test.go b/virtcontainers/acrn_test.go index f335833a8..12a0ea5fd 100644 --- a/virtcontainers/acrn_test.go +++ b/virtcontainers/acrn_test.go @@ -230,7 +230,7 @@ func TestAcrnCreateSandbox(t *testing.T) { //set PID to 1 to ignore hypercall to get UUID and set a random UUID a.state.PID = 1 a.state.UUID = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6" - err = a.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil) + err = a.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false) assert.NoError(err) assert.Exactly(acrnConfig, a.config) } diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index bebad7c3f..e6fdb0a30 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -6,8 +6,10 @@ package virtcontainers import ( + "bufio" "context" "fmt" + "io" "net" "net/http" "os" @@ -31,6 +33,7 @@ import ( "github.com/sirupsen/logrus" "github.com/blang/semver" + "github.com/containerd/console" "github.com/kata-containers/runtime/virtcontainers/device/config" fcmodels "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models" "github.com/kata-containers/runtime/virtcontainers/store" @@ -77,16 +80,12 @@ var fcKernelParams = append(commonVirtioblkKernelRootParams, []Param{ {"reboot", "k"}, {"panic", "1"}, {"iommu", "off"}, - {"8250.nr_uarts", "0"}, {"net.ifnames", "0"}, {"random.trust_cpu", "on"}, // Firecracker doesn't support ACPI // Fix kernel error "ACPI BIOS Error (bug)" {"acpi", "off"}, - - // Tell agent where to send the logs - {"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, }...) func (s vmmState) String() string { @@ -141,8 +140,9 @@ type firecracker struct { config HypervisorConfig pendingDevices []firecrackerDevice // Devices to be added when the FC API is ready - state firecrackerState - jailed bool //Set to true if jailer is enabled + state firecrackerState + jailed bool //Set to true if jailer is enabled + stateful bool //Set to true if running with shimv2 } type firecrackerDevice struct { @@ -211,7 +211,7 @@ func (fc *firecracker) bindMount(ctx context.Context, source, destination string // For firecracker this call only sets the internal structure up. // The sandbox will be created and started through startSandbox(). -func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore, stateful bool) error { fc.ctx = ctx span, _ := fc.trace("createSandbox") @@ -223,6 +223,7 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N fc.store = vcStore fc.state.set(notReady) fc.config = *hypervisorConfig + fc.stateful = stateful // When running with jailer all resources need to be under // a specific location and that location needs to have @@ -386,13 +387,17 @@ func (fc *firecracker) fcInit(timeout int) error { var args []string var cmd *exec.Cmd + if !fc.config.Debug && fc.stateful { + args = append(args, "--daemonize") + } + //https://github.com/firecracker-microvm/firecracker/blob/master/docs/jailer.md#jailer-usage //--seccomp-level specifies whether seccomp filters should be installed and how restrictive they should be. Possible values are: //0 : disabled. //1 : basic filtering. This prohibits syscalls not whitelisted by Firecracker. //2 (default): advanced filtering. This adds further checks on some of the parameters of the allowed syscalls. if fc.jailed { - args = []string{ + args = append(args, "--id", fc.id, "--node", "0", //FIXME: Comprehend NUMA topology or explicit ignore "--seccomp-level", "2", @@ -400,8 +405,7 @@ func (fc *firecracker) fcInit(timeout int) error { "--uid", "0", //https://github.com/kata-containers/runtime/issues/1869 "--gid", "0", "--chroot-base-dir", fc.chrootBaseDir, - "--daemonize", - } + ) if fc.netNSPath != "" { args = append(args, "--netns", fc.netNSPath) } @@ -412,6 +416,16 @@ func (fc *firecracker) fcInit(timeout int) error { } + if fc.config.Debug && fc.stateful { + stdin, err := fc.watchConsole() + if err != nil { + return err + } + + cmd.Stderr = stdin + cmd.Stdout = stdin + } + fc.Logger().WithField("hypervisor args", args).Debug() fc.Logger().WithField("hypervisor cmd", cmd).Debug() if err := cmd.Start(); err != nil { @@ -660,6 +674,16 @@ func (fc *firecracker) startSandbox(timeout int) error { return err } + if fc.config.Debug && fc.stateful { + fcKernelParams = append(fcKernelParams, Param{"console", "ttyS0"}) + } else { + fcKernelParams = append(fcKernelParams, []Param{ + {"8250.nr_uarts", "0"}, + // Tell agent where to send the logs + {"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)}, + }...) + } + kernelParams := append(fc.config.KernelParams, fcKernelParams...) strParams := SerializeParams(kernelParams, "=") formattedParams := strings.Join(strParams, " ") @@ -1097,3 +1121,37 @@ func (fc *firecracker) generateSocket(id string, useVsock bool) (interface{}, er Port: uint32(vSockPort), }, nil } + +func (fc *firecracker) watchConsole() (*os.File, error) { + master, slave, err := console.NewPty() + if err != nil { + fc.Logger().WithField("Error create pseudo tty", err).Debug() + return nil, err + } + + stdio, err := os.OpenFile(slave, syscall.O_RDWR, 0700) + if err != nil { + fc.Logger().WithError(err).Debugf("open pseudo tty %s", slave) + return nil, err + } + + go func() { + scanner := bufio.NewScanner(master) + for scanner.Scan() { + fc.Logger().WithFields(logrus.Fields{ + "sandbox": fc.id, + "vmconsole": scanner.Text(), + }).Infof("reading guest console") + } + + if err := scanner.Err(); err != nil { + if err == io.EOF { + fc.Logger().Info("console watcher quits") + } else { + fc.Logger().WithError(err).Error("Failed to read guest console") + } + } + }() + + return stdio, nil +} diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 82bf44b65..37cd15a51 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -726,7 +726,7 @@ func generateVMSocket(id string, useVsock bool) (interface{}, error) { // hypervisor is the virtcontainers hypervisor interface. // The default hypervisor implementation is Qemu. type hypervisor interface { - createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error + createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error startSandbox(timeout int) error stopSandbox() error pauseSandbox() error diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index 30962e87a..30bd38cb4 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -27,7 +27,7 @@ func (m *mockHypervisor) hypervisorConfig() HypervisorConfig { return HypervisorConfig{} } -func (m *mockHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { +func (m *mockHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, store *store.VCStore, stateful bool) error { err := hypervisorConfig.valid() if err != nil { return err diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go index 799f50275..10c6a90cd 100644 --- a/virtcontainers/mock_hypervisor_test.go +++ b/virtcontainers/mock_hypervisor_test.go @@ -31,7 +31,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) { ctx := context.Background() // wrong config - err := m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil) + err := m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false) assert.Error(err) sandbox.config.HypervisorConfig = HypervisorConfig{ @@ -40,7 +40,7 @@ func TestMockHypervisorCreateSandbox(t *testing.T) { HypervisorPath: fmt.Sprintf("%s/%s", testDir, testHypervisor), } - err = m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil) + err = m.createSandbox(ctx, sandbox.config.ID, NetworkNamespace{}, &sandbox.config.HypervisorConfig, nil, false) assert.NoError(err) } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 95b0b0fa0..b23168fe5 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -463,7 +463,7 @@ func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memo } // createSandbox is the Hypervisor sandbox creation implementation for govmmQemu. -func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore) error { +func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, vcStore *store.VCStore, stateful bool) error { // Save the tracing context q.ctx = ctx diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 1f0d988b7..a503bb4f7 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -99,7 +99,7 @@ func TestQemuCreateSandbox(t *testing.T) { parentDir := store.SandboxConfigurationRootPath(sandbox.id) assert.NoError(os.MkdirAll(parentDir, store.DirMode)) - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) assert.NoError(err) assert.NoError(os.RemoveAll(parentDir)) assert.Exactly(qemuConfig, q.config) @@ -131,7 +131,7 @@ func TestQemuCreateSandboxMissingParentDirFail(t *testing.T) { parentDir := store.SandboxConfigurationRootPath(sandbox.id) assert.NoError(os.RemoveAll(parentDir)) - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) assert.NoError(err) } @@ -429,7 +429,7 @@ func TestQemuFileBackedMem(t *testing.T) { q := &qemu{} sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) assert.NoError(err) assert.Equal(q.qemuConfig.Knobs.FileBackedMem, true) @@ -445,7 +445,7 @@ func TestQemuFileBackedMem(t *testing.T) { sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS sandbox.config.HypervisorConfig.MemoryPath = fallbackFileBackedMemDir - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) expectErr := errors.New("VM templating has been enabled with either virtio-fs or file backed memory and this configuration will not work") assert.Equal(expectErr.Error(), err.Error()) @@ -456,7 +456,7 @@ func TestQemuFileBackedMem(t *testing.T) { q = &qemu{} sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc" - err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store) + err = q.createSandbox(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig, sandbox.store, false) assert.NoError(err) assert.Equal(q.qemuConfig.Knobs.FileBackedMem, false) assert.Equal(q.qemuConfig.Knobs.MemShared, false) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 9771f0d24..d8d548ef3 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -569,7 +569,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor s.Restore() // new store doesn't require hypervisor to be stored immediately - if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil); err != nil { + if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, nil, s.stateful); err != nil { return nil, err } } else { @@ -591,7 +591,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor s.state = state } - if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store); err != nil { + if err = s.hypervisor.createSandbox(ctx, s.id, s.networkNS, &sandboxConfig.HypervisorConfig, s.store, s.stateful); err != nil { return nil, err } } diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go index 77f85450c..4b77c22de 100644 --- a/virtcontainers/vm.go +++ b/virtcontainers/vm.go @@ -172,7 +172,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { } }() - if err = hypervisor.createSandbox(ctx, id, NetworkNamespace{}, &config.HypervisorConfig, vcStore); err != nil { + if err = hypervisor.createSandbox(ctx, id, NetworkNamespace{}, &config.HypervisorConfig, vcStore, false); err != nil { return nil, err }