runtime: Support the remote hypervisor type

This patch adds the support of the remote hypervisor type.
Shim opens a Unix domain socket specified in the config file,
and sends TTPRC requests to a external process to control
sandbox VMs.

Fixes #4482

Co-authored-by: Pradipta Banerjee <pradipta.banerjee@gmail.com>
Co-authored-by: stevenhorsman <steven@uk.ibm.com>
Signed-off-by: Yohei Ueda <yohei@jp.ibm.com>
(based on commit f9278f22c3)

src/runtime/cmd/kata-runtime/kata-check_amd64.go

@@ -323,6 +323,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
 		return kvmIsUsable()
 	case vc.AcrnHypervisor:
 		return acrnIsUsable()
+	case vc.RemoteHypervisor:
+		return nil
 	case vc.MockHypervisor:
 		return nil
 	default:

src/runtime/cmd/kata-runtime/kata-check_arm64.go

@@ -86,6 +86,9 @@ func checkKVMExtensions() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	if err := kvmIsUsable(); err != nil {
 		return err
 	}

src/runtime/cmd/kata-runtime/kata-check_ppc64le.go

@@ -61,6 +61,9 @@ func setCPUtype(hypervisorType vc.HypervisorType) error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/cmd/kata-runtime/kata-check_s390x.go

@@ -55,6 +55,9 @@ func kvmIsUsable() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/pkg/katautils/config.go

@@ -53,6 +53,7 @@ const (
 	acrnHypervisorTableType        = "acrn"
 	dragonballHypervisorTableType  = "dragonball"
 	stratovirtHypervisorTableType  = "stratovirt"
+	remoteHypervisorTableType      = "remote"
 
 	// the maximum amount of PCI bridges that can be cold plugged in a VM
 	maxPCIBridges uint32 = 5
@@ -105,6 +106,7 @@ type hypervisor struct {
 	GuestMemoryDumpPath            string                    `toml:"guest_memory_dump_path"`
 	SeccompSandbox                 string                    `toml:"seccompsandbox"`
 	BlockDeviceAIO                 string                    `toml:"block_device_aio"`
+	RemoteHypervisorSocket         string                    `toml:"remote_hypervisor_socket"`
 	HypervisorPathList             []string                  `toml:"valid_hypervisor_paths"`
 	JailerPathList                 []string                  `toml:"valid_jailer_paths"`
 	CtlPathList                    []string                  `toml:"valid_ctlpaths"`
@@ -134,6 +136,7 @@ type hypervisor struct {
 	MemSlots                       uint32                    `toml:"memory_slots"`
 	DefaultBridges                 uint32                    `toml:"default_bridges"`
 	Msize9p                        uint32                    `toml:"msize_9p"`
+	RemoteHypervisorTimeout        uint32                    `toml:"remote_hypervisor_timeout"`
 	NumVCPUs                       float32                   `toml:"default_vcpus"`
 	BlockDeviceCacheSet            bool                      `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect         bool                      `toml:"block_device_cache_direct"`
@@ -1242,6 +1245,14 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	}, nil
 }
 
+func newRemoteHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
+
+	return vc.HypervisorConfig{
+		RemoteHypervisorSocket:  h.RemoteHypervisorSocket,
+		RemoteHypervisorTimeout: h.RemoteHypervisorTimeout,
+	}, nil
+}
+
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
 	if f.TemplatePath == "" {
 		f.TemplatePath = defaultTemplatePath
@@ -1281,6 +1292,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
 		case stratovirtHypervisorTableType:
 			config.HypervisorType = vc.StratovirtHypervisor
 			hConfig, err = newStratovirtHypervisorConfig(hypervisor)
+		case remoteHypervisorTableType:
+			config.HypervisorType = vc.RemoteHypervisor
+			hConfig, err = newRemoteHypervisorConfig(hypervisor)
 		default:
 			err = fmt.Errorf("%s: %+q", errInvalidHypervisorPrefix, k)
 		}
@@ -1882,6 +1896,11 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 // checkHypervisorConfig performs basic "sanity checks" on the hypervisor
 // config.
 func checkHypervisorConfig(config vc.HypervisorConfig) error {
+
+	if config.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	type image struct {
 		path   string
 		initrd bool

src/runtime/pkg/katautils/config_test.go

@@ -1564,6 +1564,17 @@ func TestCheckHypervisorConfig(t *testing.T) {
 		// reset logger
 		kataUtilsLogger.Logger.Out = savedOut
 	}
+
+	// Check remote hypervisor doesn't error with missing unnescessary config
+	remoteConfig := vc.HypervisorConfig{
+		RemoteHypervisorSocket: "dummy_socket",
+		ImagePath:              "",
+		InitrdPath:             "",
+		MemorySize:             0,
+	}
+
+	err := checkHypervisorConfig(remoteConfig)
+	assert.NoError(err, "remote hypervisor config")
 }
 
 func TestCheckNetNsConfig(t *testing.T) {

src/runtime/protocols/hypervisor/hypervisor.proto

@@ -1,3 +1,6 @@
+// (C) Copyright IBM Corp. 2022.
+// SPDX-License-Identifier: Apache-2.0
+
 syntax = "proto3";
 
 option go_package = "./";

src/runtime/virtcontainers/hypervisor.go

@@ -57,6 +57,9 @@ const (
 	// VirtFrameworkHypervisor is the Darwin Virtualization.framework hypervisor
 	VirtframeworkHypervisor HypervisorType = "virtframework"
 
+	// RemoteHypervisor is the Remote hypervisor.
+	RemoteHypervisor HypervisorType = "remote"
+
 	// MockHypervisor is a mock hypervisor for testing purposes
 	MockHypervisor HypervisorType = "mock"
 
@@ -240,6 +243,9 @@ func (hType *HypervisorType) Set(value string) error {
 	case "virtframework":
 		*hType = VirtframeworkHypervisor
 		return nil
+	case "remote":
+		*hType = RemoteHypervisor
+		return nil
 	case "mock":
 		*hType = MockHypervisor
 		return nil
@@ -261,6 +267,8 @@ func (hType *HypervisorType) String() string {
 		return string(ClhHypervisor)
 	case StratovirtHypervisor:
 		return string(StratovirtHypervisor)
+	case RemoteHypervisor:
+		return string(RemoteHypervisor)
 	case MockHypervisor:
 		return string(MockHypervisor)
 	default:
@@ -455,6 +463,15 @@ type HypervisorConfig struct {
 	// BlockiDeviceAIO specifies the I/O API to be used.
 	BlockDeviceAIO string
 
+	// The socket to connect to the remote hypervisor implementation on
+	RemoteHypervisorSocket string
+
+	// The name of the sandbox (pod)
+	SandboxName string
+
+	// The name of the namespace of the sandbox (pod)
+	SandboxNamespace string
+
 	// The user maps to the uid.
 	User string
 
@@ -563,6 +580,9 @@ type HypervisorConfig struct {
 	// Group ID.
 	Gid uint32
 
+	// Timeout for actions e.g. startVM for the remote hypervisor
+	RemoteHypervisorTimeout uint32
+
 	// BlockDeviceCacheSet specifies cache-related options will be set to block devices or not.
 	BlockDeviceCacheSet bool
 

src/runtime/virtcontainers/hypervisor_config_darwin.go

@@ -11,6 +11,10 @@ import (
 
 func validateHypervisorConfig(conf *HypervisorConfig) error {
 
+	if conf.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	if conf.KernelPath == "" {
 		return fmt.Errorf("Missing kernel path")
 	}

src/runtime/virtcontainers/hypervisor_config_linux.go

@@ -13,6 +13,10 @@ import (
 
 func validateHypervisorConfig(conf *HypervisorConfig) error {
 
+	if conf.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	if conf.KernelPath == "" {
 		return fmt.Errorf("Missing kernel path")
 	}

src/runtime/virtcontainers/hypervisor_config_test.go

@@ -28,3 +28,12 @@ func TestHypervisorConfigNoKernelPath(t *testing.T) {
 
 	testHypervisorConfigValid(t, hypervisorConfig, false)
 }
+
+func TestRemoteHypervisorConfigNoKernelPath(t *testing.T) {
+	hypervisorConfig := &HypervisorConfig{
+		RemoteHypervisorSocket: "dummy_socket",
+		KernelPath:             "",
+	}
+
+	testHypervisorConfigValid(t, hypervisorConfig, true)
+}

src/runtime/virtcontainers/hypervisor_linux.go

@@ -40,6 +40,8 @@ func NewHypervisor(hType HypervisorType) (Hypervisor, error) {
 		return &stratovirt{}, nil
 	case DragonballHypervisor:
 		return &mockHypervisor{}, nil
+	case RemoteHypervisor:
+		return &remoteHypervisor{}, nil
 	case MockHypervisor:
 		return &mockHypervisor{}, nil
 	default:

src/runtime/virtcontainers/hypervisor_test.go

@@ -7,11 +7,12 @@ package virtcontainers
 
 import (
 	"fmt"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	"github.com/stretchr/testify/assert"
 	"os"
 	"strings"
 	"testing"
+
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestGetKernelRootParams(t *testing.T) {
@@ -186,6 +187,10 @@ func TestSetMockHypervisorType(t *testing.T) {
 	testSetHypervisorType(t, "mock", MockHypervisor)
 }
 
+func TestSetRemoteHypervisorType(t *testing.T) {
+	testSetHypervisorType(t, "remote", RemoteHypervisor)
+}
+
 func TestSetUnknownHypervisorType(t *testing.T) {
 	var hypervisorType HypervisorType
 	assert := assert.New(t)
@@ -207,6 +212,11 @@ func TestStringFromQemuHypervisorType(t *testing.T) {
 	testStringFromHypervisorType(t, hypervisorType, "qemu")
 }
 
+func TestStringFromRemoteHypervisorType(t *testing.T) {
+	hypervisorType := RemoteHypervisor
+	testStringFromHypervisorType(t, hypervisorType, "remote")
+}
+
 func TestStringFromMockHypervisorType(t *testing.T) {
 	hypervisorType := MockHypervisor
 	testStringFromHypervisorType(t, hypervisorType, "mock")
@@ -224,6 +234,12 @@ func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType Hypervisor
 	assert.Exactly(hy, expected)
 }
 
+func TestNewHypervisorFromRemoteHypervisorType(t *testing.T) {
+	hypervisorType := RemoteHypervisor
+	expectedHypervisor := &remoteHypervisor{}
+	testNewHypervisorFromHypervisorType(t, hypervisorType, expectedHypervisor)
+}
+
 func TestNewHypervisorFromMockHypervisorType(t *testing.T) {
 	hypervisorType := MockHypervisor
 	expectedHypervisor := &mockHypervisor{}

src/runtime/virtcontainers/kata_agent.go

@@ -78,9 +78,13 @@ const (
 	defaultSeLinuxContainerType = "container_t"
 )
 
+type customRequestTimeoutKeyType struct{}
+
 var (
 	checkRequestTimeout           = 30 * time.Second
 	defaultRequestTimeout         = 60 * time.Second
+	remoteRequestTimeout          = 300 * time.Second
+	customRequestTimeoutKey       = customRequestTimeoutKeyType(struct{}{})
 	errorMissingOCISpec           = errors.New("Missing OCI specification")
 	defaultKataHostSharedDir      = "/run/kata-containers/shared/sandboxes/"
 	defaultKataGuestSharedDir     = "/run/kata-containers/shared/containers/"
@@ -376,6 +380,8 @@ func (k *kataAgent) agentURL() (string, error) {
 		return s.String(), nil
 	case types.HybridVSock:
 		return s.String(), nil
+	case types.RemoteSock:
+		return s.String(), nil
 	case types.MockHybridVSock:
 		return s.String(), nil
 	default:
@@ -426,6 +432,7 @@ func (k *kataAgent) configure(ctx context.Context, h Hypervisor, id, sharePath s
 		if err != nil {
 			return err
 		}
+	case types.RemoteSock:
 	case types.MockHybridVSock:
 	default:
 		return types.ErrInvalidConfigType
@@ -745,37 +752,43 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error {
 		return err
 	}
 
-	// Check grpc server is serving
-	if err = k.check(ctx); err != nil {
-		return err
-	}
+	var kmodules []*grpc.KernelModule
 
-	// If a Policy has been specified, send it to the agent.
-	if len(sandbox.config.AgentConfig.Policy) > 0 {
-		if err := sandbox.agent.setPolicy(ctx, sandbox.config.AgentConfig.Policy); err != nil {
+	if sandbox.config.HypervisorType == RemoteHypervisor {
+		ctx = context.WithValue(ctx, customRequestTimeoutKey, remoteRequestTimeout)
+	} else {
+		// Check grpc server is serving
+		if err = k.check(ctx); err != nil {
 			return err
 		}
-	}
 
-	// Setup network interfaces and routes
-	interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network)
-	if err != nil {
-		return err
-	}
-	if err = k.updateInterfaces(ctx, interfaces); err != nil {
-		return err
-	}
-	if _, err = k.updateRoutes(ctx, routes); err != nil {
-		return err
-	}
-	if err = k.addARPNeighbors(ctx, neighs); err != nil {
-		return err
+		// If a Policy has been specified, send it to the agent.
+		if len(sandbox.config.AgentConfig.Policy) > 0 {
+			if err := sandbox.agent.setPolicy(ctx, sandbox.config.AgentConfig.Policy); err != nil {
+				return err
+			}
+		}
+
+		// Setup network interfaces and routes
+		interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network)
+		if err != nil {
+			return err
+		}
+		if err = k.updateInterfaces(ctx, interfaces); err != nil {
+			return err
+		}
+		if _, err = k.updateRoutes(ctx, routes); err != nil {
+			return err
+		}
+		if err = k.addARPNeighbors(ctx, neighs); err != nil {
+			return err
+		}
+
+		kmodules = setupKernelModules(k.kmodules)
 	}
 
 	storages := setupStorages(ctx, sandbox)
 
-	kmodules := setupKernelModules(k.kmodules)
-
 	req := &grpc.CreateSandboxRequest{
 		Hostname:      hostname,
 		Dns:           dns,
@@ -2104,7 +2117,12 @@ func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx c
 	case grpcCheckRequest:
 		newCtx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
 	default:
-		newCtx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
+		var requestTimeout = defaultRequestTimeout
+
+		if timeout, ok := ctx.Value(customRequestTimeoutKey).(time.Duration); ok {
+			requestTimeout = timeout
+		}
+		newCtx, cancel = context.WithTimeout(ctx, requestTimeout)
 	}
 
 	return newCtx, cancel

src/runtime/virtcontainers/pkg/agent/protocols/client/client.go

@@ -34,6 +34,7 @@ import (
 const (
 	VSockSocketScheme     = "vsock"
 	HybridVSockScheme     = "hvsock"
+	RemoteSockScheme      = "remote"
 	MockHybridVSockScheme = "mock"
 )
 
@@ -235,6 +236,11 @@ func parse(sock string) (string, *url.URL, error) {
 		}
 		hybridVSockPort = uint32(port)
 		grpcAddr = HybridVSockScheme + ":" + hvsocket[0]
+	case RemoteSockScheme:
+		if addr.Host != "" {
+			return "", nil, grpcStatus.Errorf(codes.InvalidArgument, "Invalid remote sock scheme: host address must be empty: %s", sock)
+		}
+		grpcAddr = RemoteSockScheme + ":" + addr.Path
 	// just for tests use.
 	case MockHybridVSockScheme:
 		if addr.Path == "" {
@@ -255,6 +261,8 @@ func agentDialer(addr *url.URL) dialer {
 		return VsockDialer
 	case HybridVSockScheme:
 		return HybridVSockDialer
+	case RemoteSockScheme:
+		return RemoteSockDialer
 	case MockHybridVSockScheme:
 		return MockHybridVSockDialer
 	default:
@@ -435,6 +443,31 @@ func HybridVSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
 	return commonDialer(timeout, dialFunc, timeoutErr)
 }
 
+// RemoteSockDialer dials to an agent in a remote hypervisor sandbox
+func RemoteSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
+
+	s := strings.Split(sock, ":")
+	if !(len(s) == 2 && s[0] == RemoteSockScheme) {
+		return nil, fmt.Errorf("failed to parse remote sock: %q", sock)
+	}
+	socketPath := s[1]
+
+	logrus.Printf("Dialing remote sock: %q %q", socketPath, sock)
+
+	dialFunc := func() (net.Conn, error) {
+		conn, err := net.Dial("unix", socketPath)
+		if err != nil {
+			logrus.Errorf("failed to dial remote sock %q: %v", socketPath, err)
+			return nil, err
+		}
+		return conn, nil
+	}
+
+	timeoutErr := grpcStatus.Errorf(codes.DeadlineExceeded, "timed out connecting to remote sock: %s", socketPath)
+
+	return commonDialer(timeout, dialFunc, timeoutErr)
+}
+
 // just for tests use.
 func MockHybridVSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
 	sock = strings.TrimPrefix(sock, "mock:")

src/runtime/virtcontainers/remote.go

@@ -0,0 +1,288 @@
+// Copyright (c) 2022 IBM Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+package virtcontainers
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	cri "github.com/containerd/containerd/pkg/cri/annotations"
+	persistapi "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const defaultMinTimeout = 60
+
+type remoteHypervisor struct {
+	sandboxID       remoteHypervisorSandboxID
+	agentSocketPath string
+	config          HypervisorConfig
+}
+
+type remoteHypervisorSandboxID string
+
+type remoteService struct {
+	conn   *grpc.ClientConn
+	client pb.HypervisorClient
+}
+
+func openRemoteService(socketPath string) (*remoteService, error) {
+
+	conn, err := grpc.Dial(fmt.Sprintf("unix://%s", socketPath), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to remote hypervisor socket: %w", err)
+	}
+	defer conn.Close()
+	client := pb.NewHypervisorClient(conn)
+
+	s := &remoteService{
+		conn:   conn,
+		client: client,
+	}
+
+	return s, nil
+}
+
+func (s *remoteService) Close() error {
+	return s.conn.Close()
+}
+
+func (rh *remoteHypervisor) CreateVM(ctx context.Context, id string, network Network, hypervisorConfig *HypervisorConfig) error {
+
+	rh.sandboxID = remoteHypervisorSandboxID(id)
+
+	if err := rh.setConfig(hypervisorConfig); err != nil {
+		return err
+	}
+
+	s, err := openRemoteService(hypervisorConfig.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	annotations := map[string]string{}
+	annotations[cri.SandboxName] = hypervisorConfig.SandboxName
+	annotations[cri.SandboxNamespace] = hypervisorConfig.SandboxNamespace
+
+	req := &pb.CreateVMRequest{
+		Id:                   id,
+		Annotations:          annotations,
+		NetworkNamespacePath: network.NetworkID(),
+	}
+
+	res, err := s.client.CreateVM(ctx, req)
+	if err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	if res.AgentSocketPath == "" {
+		return errors.New("remote hypervisor does not return tunnel socket path")
+	}
+
+	rh.agentSocketPath = res.AgentSocketPath
+
+	return nil
+}
+
+func (rh *remoteHypervisor) StartVM(ctx context.Context, timeout int) error {
+
+	minTimeout := defaultMinTimeout
+	if rh.config.RemoteHypervisorTimeout > 0 {
+		minTimeout = int(rh.config.RemoteHypervisorTimeout)
+	}
+
+	if timeout < minTimeout {
+		timeout = minTimeout
+	}
+
+	s, err := openRemoteService(rh.config.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	req := &pb.StartVMRequest{
+		Id: string(rh.sandboxID),
+	}
+
+	ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+	defer cancel()
+
+	logrus.Printf("calling remote hypervisor StartVM (timeout: %d)", timeout)
+
+	if _, err := s.client.StartVM(ctx2, req); err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	return nil
+}
+
+func (rh *remoteHypervisor) AttestVM(ctx context.Context) error {
+	return nil
+}
+
+func (rh *remoteHypervisor) StopVM(ctx context.Context, waitOnly bool) error {
+
+	s, err := openRemoteService(rh.config.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	req := &pb.StopVMRequest{
+		Id: string(rh.sandboxID),
+	}
+
+	if _, err := s.client.StopVM(ctx, req); err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	return nil
+}
+
+func (rh *remoteHypervisor) GenerateSocket(id string) (interface{}, error) {
+
+	socketPath := rh.agentSocketPath
+	if len(socketPath) == 0 {
+		return nil, errors.New("failed to generate remote sock: TunnelSocketPath is not set")
+	}
+
+	remoteSock := types.RemoteSock{
+		SandboxID:        id,
+		TunnelSocketPath: socketPath,
+	}
+
+	return remoteSock, nil
+}
+
+func notImplemented(name string) error {
+
+	err := errors.Errorf("%s: not implemented", name)
+
+	logrus.Errorf(err.Error())
+
+	if tracer, ok := err.(interface{ StackTrace() errors.StackTrace }); ok {
+		for _, f := range tracer.StackTrace() {
+			logrus.Errorf("%+s:%d\n", f, f)
+		}
+	}
+
+	return err
+}
+
+func (rh *remoteHypervisor) PauseVM(ctx context.Context) error {
+	return notImplemented("PauseVM")
+}
+
+func (rh *remoteHypervisor) SaveVM() error {
+	return notImplemented("SaveVM")
+}
+
+func (rh *remoteHypervisor) ResumeVM(ctx context.Context) error {
+	return notImplemented("ResumeVM")
+}
+
+func (rh *remoteHypervisor) AddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) error {
+	// TODO should we return notImplemented("AddDevice"), rather than nil and ignoring it?
+	logrus.Printf("addDevice: deviceType=%v devInfo=%#v", devType, devInfo)
+	return nil
+}
+
+func (rh *remoteHypervisor) HotplugAddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) {
+	return nil, notImplemented("HotplugAddDevice")
+}
+
+func (rh *remoteHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) {
+	return nil, notImplemented("HotplugRemoveDevice")
+}
+
+func (rh *remoteHypervisor) ResizeMemory(ctx context.Context, memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, MemoryDevice, error) {
+	return memMB, MemoryDevice{}, nil
+}
+
+func (rh *remoteHypervisor) GetTotalMemoryMB(ctx context.Context) uint32 {
+	//The remote hypervisor uses the peer pod config to determine the memory of the VM, so we need to use static resource management
+	logrus.Error("GetTotalMemoryMB - remote hypervisor cannot update resources")
+	return 0
+}
+
+func (rh *remoteHypervisor) ResizeVCPUs(ctx context.Context, vcpus uint32) (uint32, uint32, error) {
+	return vcpus, vcpus, nil
+}
+
+func (rh *remoteHypervisor) GetVMConsole(ctx context.Context, sandboxID string) (string, string, error) {
+	return "", "", notImplemented("GetVMConsole")
+}
+
+func (rh *remoteHypervisor) Disconnect(ctx context.Context) {
+	notImplemented("Disconnect")
+}
+
+func (rh *remoteHypervisor) Capabilities(ctx context.Context) types.Capabilities {
+	var caps types.Capabilities
+	caps.SetBlockDeviceHotplugSupport()
+	return caps
+}
+
+func (rh *remoteHypervisor) HypervisorConfig() HypervisorConfig {
+	return rh.config
+}
+
+func (rh *remoteHypervisor) GetThreadIDs(ctx context.Context) (VcpuThreadIDs, error) {
+	// Not supported. return success
+	// Just allocating an empty map
+	return VcpuThreadIDs{}, nil
+}
+
+func (rh *remoteHypervisor) Cleanup(ctx context.Context) error {
+	return nil
+}
+
+func (rh *remoteHypervisor) setConfig(config *HypervisorConfig) error {
+	// Create a Validator specific for remote hypervisor
+	rh.config = *config
+
+	return nil
+}
+
+func (rh *remoteHypervisor) GetPids() []int {
+	// let's use shim pid as it used by crio to fetch start time
+	return []int{os.Getpid()}
+}
+
+func (rh *remoteHypervisor) GetVirtioFsPid() *int {
+	panic(notImplemented("GetVirtioFsPid"))
+}
+
+func (rh *remoteHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error {
+	panic(notImplemented("fromGrpc"))
+}
+
+func (rh *remoteHypervisor) toGrpc(ctx context.Context) ([]byte, error) {
+	panic(notImplemented("toGrpc"))
+}
+
+func (rh *remoteHypervisor) Check() error {
+	return nil
+}
+
+func (rh *remoteHypervisor) Save() persistapi.HypervisorState {
+	return persistapi.HypervisorState{}
+}
+
+func (rh *remoteHypervisor) Load(persistapi.HypervisorState) {
+	notImplemented("Load")
+}
+
+func (rh *remoteHypervisor) IsRateLimiterBuiltin() bool {
+	return false
+}

src/runtime/virtcontainers/remote_test.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2023 IBM Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+package virtcontainers
+
+import (
+	"testing"
+
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func newRemoteConfig() HypervisorConfig {
+	return HypervisorConfig{
+		RemoteHypervisorSocket:  "/run/peerpod/hypervisor.sock",
+		RemoteHypervisorTimeout: 600,
+		DisableGuestSeLinux:     true,
+		EnableAnnotations:       []string{},
+	}
+}
+
+func TestRemoteHypervisorGenerateSocket(t *testing.T) {
+	assert := assert.New(t)
+
+	remoteHypervisor := remoteHypervisor{
+		config: newRemoteConfig(),
+	}
+	id := "sandboxId"
+
+	// No socketPath should error
+	_, err := remoteHypervisor.GenerateSocket(id)
+	assert.Error(err)
+
+	socketPath := "socketPath"
+	remoteHypervisor.agentSocketPath = socketPath
+
+	result, err := remoteHypervisor.GenerateSocket(id)
+	assert.NoError(err)
+
+	expected := types.RemoteSock{
+		SandboxID:        id,
+		TunnelSocketPath: socketPath,
+	}
+	assert.Equal(result, expected)
+}

src/runtime/virtcontainers/sandbox.go

@@ -28,6 +28,8 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 
+	cri "github.com/containerd/containerd/pkg/cri/annotations"
+	crio "github.com/containers/podman/v4/pkg/annotations"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
@@ -635,6 +637,8 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 
 	}
 
+	setHypervisorConfigAnnotations(&sandboxConfig)
+
 	coldPlugVFIO, err := s.coldOrHotPlugVFIO(&sandboxConfig)
 	if err != nil {
 		return nil, err
@@ -722,6 +726,23 @@ func (s *Sandbox) coldOrHotPlugVFIO(sandboxConfig *SandboxConfig) (bool, error)
 	return coldPlugVFIO, nil
 }
 
+func setHypervisorConfigAnnotations(sandboxConfig *SandboxConfig) {
+	if len(sandboxConfig.Containers) > 0 {
+		// These values are required by remote hypervisor
+		for _, a := range []string{cri.SandboxName, crio.SandboxName} {
+			if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok {
+				sandboxConfig.HypervisorConfig.SandboxName = value
+			}
+		}
+
+		for _, a := range []string{cri.SandboxNamespace, crio.Namespace} {
+			if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok {
+				sandboxConfig.HypervisorConfig.SandboxNamespace = value
+			}
+		}
+	}
+}
+
 func (s *Sandbox) createResourceController() error {
 	var err error
 	cgroupPath := ""

src/runtime/virtcontainers/types/sandbox.go

@@ -7,6 +7,7 @@ package types
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"strings"
 
@@ -37,6 +38,7 @@ const (
 	HybridVSockScheme     = "hvsock"
 	MockHybridVSockScheme = "mock"
 	VSockScheme           = "vsock"
+	RemoteSockScheme      = "remote"
 )
 
 // SandboxState is a sandbox state structure
@@ -210,6 +212,16 @@ func (s *HybridVSock) String() string {
 	return fmt.Sprintf("%s://%s:%d", HybridVSockScheme, s.UdsPath, s.Port)
 }
 
+type RemoteSock struct {
+	Conn             net.Conn
+	SandboxID        string
+	TunnelSocketPath string
+}
+
+func (s *RemoteSock) String() string {
+	return fmt.Sprintf("%s://%s", RemoteSockScheme, s.TunnelSocketPath)
+}
+
 // MockHybridVSock defines a mock hybrid vsocket for tests only.
 type MockHybridVSock struct {
 	UdsPath string

src/runtime/virtcontainers/types/sandbox_test.go

@@ -6,6 +6,8 @@
 package types
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -163,6 +165,78 @@ func TestVolumesStringSuccessful(t *testing.T) {
 	assert.Equal(t, result, expected)
 }
 
+func TestStringFromVSock(t *testing.T) {
+	assert := assert.New(t)
+
+	dir := t.TempDir()
+
+	contextID := uint64(16187)
+	port := uint32(1024)
+	vsockFilename := filepath.Join(dir, "vsock")
+
+	vsockFile, err := os.Create(vsockFilename)
+	assert.NoError(err)
+	defer vsockFile.Close()
+
+	vsock := VSock{
+		ContextID: contextID,
+		Port:      port,
+		VhostFd:   vsockFile,
+	}
+
+	expected := "vsock://16187:1024"
+
+	assert.Equal(vsock.String(), expected)
+}
+
+func TestStringFromHybridVSock(t *testing.T) {
+	assert := assert.New(t)
+
+	udsPath := "udspath"
+	contextID := uint64(16187)
+	port := uint32(1024)
+
+	sock := HybridVSock{
+		UdsPath:   udsPath,
+		ContextID: contextID,
+		Port:      port,
+	}
+
+	expected := "hvsock://udspath:1024"
+
+	assert.Equal(sock.String(), expected)
+}
+
+func TestStringFromRemoteSock(t *testing.T) {
+	assert := assert.New(t)
+
+	sandboxID := "sandboxID"
+	tunnelSockerPath := "tunnelSocketPath"
+
+	sock := RemoteSock{
+		SandboxID:        sandboxID,
+		TunnelSocketPath: tunnelSockerPath,
+	}
+
+	expected := "remote://tunnelSocketPath"
+
+	assert.Equal(sock.String(), expected)
+}
+
+func TestStringFromMockHybridVSock(t *testing.T) {
+	assert := assert.New(t)
+
+	udsPath := "udspath"
+
+	sock := MockHybridVSock{
+		UdsPath: udsPath,
+	}
+
+	expected := "mock://udspath"
+
+	assert.Equal(sock.String(), expected)
+}
+
 func TestSocketsSetSuccessful(t *testing.T) {
 	sockets := &Sockets{}