aljaz/kata-containers
1
0
Fork 0
mirror of https://github.com/aljazceru/kata-containers.git synced 2026-02-23 15:34:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

runtime: add reconnect timeout for vhost user block

Browse Source 
Fixes: #6075
Signed-off-by: zhaojizhuang <571130360@qq.com>
This commit is contained in:
zhaojizhuang
2023-01-17 11:23:36 +08:00
committed by zhaojizhuang
parent 95602c8c08
commit ca02c9f512
21 changed files with 144 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/runtime/pkg/device/config/config.go
View File

@@ -87,6 +87,8 @@ const (
// Define the string key for DriverOptions in DeviceInfo struct
FsTypeOpt = "fstype"
BlockDriverOpt = "block-driver"
VhostUserReconnectTimeOutOpt = "vhost-user-reconnect-timeout"
)
const (
@@ -97,6 +99,15 @@ const (
VhostUserSCSIMajor = 242
)
const (
// The timeout for reconnecting on non-server sockets when the remote end
// goes away.
// qemu will delay this many seconds and then attempt to reconnect. Zero
// disables reconnecting, and is the default.
DefaultVhostUserReconnectTimeOut = 0
)
// Defining these as a variable instead of a const, to allow
// overriding this in the tests.
@@ -320,6 +331,9 @@ type VhostUserDeviceAttrs struct {
CacheSize uint32
QueueSize uint32
// Reconnect timeout for socket of vhost user block device
ReconnectTime uint32
}
// GetHostPathFunc is function pointer used to mock GetHostPath in tests.

37
src/runtime/pkg/device/drivers/vhost_user_blk.go
View File

@@ -8,6 +8,7 @@ package drivers
import (
"context"
"strconv"
"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
@@ -72,17 +73,19 @@ func (device *VhostUserBlkDevice) Attach(ctx context.Context, devReceiver api.De
}
vAttrs := &config.VhostUserDeviceAttrs{
DevID: utils.MakeNameID("blk", device.DeviceInfo.ID, maxDevIDSize),
SocketPath: device.DeviceInfo.HostPath,
Type: config.VhostUserBlk,
Index: index,
DevID: utils.MakeNameID("blk", device.DeviceInfo.ID, maxDevIDSize),
SocketPath: device.DeviceInfo.HostPath,
Type: config.VhostUserBlk,
Index: index,
ReconnectTime: vhostUserReconnect(device.DeviceInfo.DriverOptions),
}
deviceLogger().WithFields(logrus.Fields{
"device": device.DeviceInfo.HostPath,
"SocketPath": vAttrs.SocketPath,
"Type": config.VhostUserBlk,
"Index": index,
"device": device.DeviceInfo.HostPath,
"SocketPath": vAttrs.SocketPath,
"Type": config.VhostUserBlk,
"Index": index,
"ReconnectTime": vAttrs.ReconnectTime,
}).Info("Attaching device")
device.VhostUserDeviceAttrs = vAttrs
@@ -93,6 +96,24 @@ func (device *VhostUserBlkDevice) Attach(ctx context.Context, devReceiver api.De
return nil
}
func vhostUserReconnect(customOptions map[string]string) uint32 {
var vhostUserReconnectTimeout uint32
if customOptions == nil {
vhostUserReconnectTimeout = config.DefaultVhostUserReconnectTimeOut
} else {
reconnectTimeoutStr := customOptions[config.VhostUserReconnectTimeOutOpt]
if reconnectTimeout, err := strconv.Atoi(reconnectTimeoutStr); err != nil {
vhostUserReconnectTimeout = config.DefaultVhostUserReconnectTimeOut
deviceLogger().WithField("reconnect", reconnectTimeoutStr).WithError(err).Warn("Failed to get reconnect timeout for vhost-user-blk device")
} else {
vhostUserReconnectTimeout = uint32(reconnectTimeout)
}
}
return vhostUserReconnectTimeout
}
func isVirtioBlkBlockDriver(customOptions map[string]string) bool {
var blockDriverOption string

13
src/runtime/pkg/device/manager/manager.go
View File

@@ -10,6 +10,7 @@ import (
"context"
"encoding/hex"
"errors"
"fmt"
"sync"
"github.com/sirupsen/logrus"
@@ -42,6 +43,8 @@ type deviceManager struct {
sync.RWMutex
vhostUserStoreEnabled bool
vhostUserReconnectTimeout uint32
}
func deviceLogger() *logrus.Entry {
@@ -49,11 +52,12 @@ func deviceLogger() *logrus.Entry {
}
// NewDeviceManager creates a deviceManager object behaved as api.DeviceManager
func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserStorePath string, devices []api.Device) api.DeviceManager {
func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserStorePath string, vhostUserReconnect uint32, devices []api.Device) api.DeviceManager {
dm := &deviceManager{
vhostUserStoreEnabled: vhostUserStoreEnabled,
vhostUserStorePath: vhostUserStorePath,
devices: make(map[string]api.Device),
vhostUserStoreEnabled: vhostUserStoreEnabled,
vhostUserStorePath: vhostUserStorePath,
vhostUserReconnectTimeout: vhostUserReconnect,
devices: make(map[string]api.Device),
}
if blockDriver == config.VirtioMmio {
dm.blockDriver = config.VirtioMmio
@@ -119,6 +123,7 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
devInfo.DriverOptions = make(map[string]string)
}
devInfo.DriverOptions[config.BlockDriverOpt] = dm.blockDriver
devInfo.DriverOptions[config.VhostUserReconnectTimeOutOpt] = fmt.Sprintf("%d", dm.vhostUserReconnectTimeout)
return drivers.NewVhostUserBlkDevice(&devInfo), nil
} else if isBlock(devInfo) {
if devInfo.DriverOptions == nil {

2
src/runtime/pkg/device/manager/manager_test.go
View File

@@ -208,7 +208,7 @@ func TestAttachBlockDevice(t *testing.T) {
}
func TestAttachDetachDevice(t *testing.T) {
dm := NewDeviceManager(config.VirtioSCSI, false, "", nil)
dm := NewDeviceManager(config.VirtioSCSI, false, "", 0, nil)
path := "/dev/hda"
deviceInfo := config.DeviceInfo{

6
src/runtime/pkg/govmm/qemu/qmp.go
View File

@@ -1524,7 +1524,7 @@ func (q *QMP) ExecuteGetFD(ctx context.Context, fdname string, fd *os.File) erro
// ExecuteCharDevUnixSocketAdd adds a character device using as backend a unix socket,
// id is an identifier for the device, path specifies the local path of the unix socket,
// wait is to block waiting for a client to connect, server specifies that the socket is a listening socket.
func (q *QMP) ExecuteCharDevUnixSocketAdd(ctx context.Context, id, path string, wait, server bool) error {
func (q *QMP) ExecuteCharDevUnixSocketAdd(ctx context.Context, id, path string, wait, server bool, reconnect uint32) error {
data := map[string]interface{}{
"server": server,
"addr": map[string]interface{}{
@@ -1540,6 +1540,10 @@ func (q *QMP) ExecuteCharDevUnixSocketAdd(ctx context.Context, id, path string,
data["wait"] = wait
}
if reconnect > 0 {
data["reconnect"] = reconnect
}
args := map[string]interface{}{
"id": id,
"backend": map[string]interface{}{

2
src/runtime/pkg/govmm/qemu/qmp_test.go
View File

@@ -1445,7 +1445,7 @@ func TestExecuteCharDevUnixSocketAdd(t *testing.T) {
cfg := QMPConfig{Logger: qmpTestLogger{}}
q := startQMPLoop(buf, cfg, connectedCh, disconnectedCh)
checkVersion(t, connectedCh)
err := q.ExecuteCharDevUnixSocketAdd(context.Background(), "foo", "foo.sock", false, true)
err := q.ExecuteCharDevUnixSocketAdd(context.Background(), "foo", "foo.sock", false, true, 1)
if err != nil {
t.Fatalf("Unexpected error %v", err)
}

1
src/runtime/pkg/katautils/config-settings.go.in
View File

@@ -84,6 +84,7 @@ const defaultGuestHookPath string = ""
const defaultVirtioFSCacheMode = "never"
const defaultDisableImageNvdimm = false
const defaultVhostUserStorePath string = "/var/run/kata-containers/vhost-user/"
const defaultVhostUserDeviceReconnect = 0
const defaultRxRateLimiterMaxRate = uint64(0)
const defaultTxRateLimiterMaxRate = uint64(0)
const defaultConfidentialGuest = false

98
src/runtime/pkg/katautils/config.go
View File

@@ -136,6 +136,7 @@ type hypervisor struct {
BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"`
BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"`
EnableVhostUserStore bool `toml:"enable_vhost_user_store"`
VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"`
DisableBlockDeviceUse bool `toml:"disable_block_device_use"`
MemPrealloc bool `toml:"enable_mem_prealloc"`
HugePages bool `toml:"enable_hugepages"`
@@ -1233,54 +1234,55 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run
func GetDefaultHypervisorConfig() vc.HypervisorConfig {
return vc.HypervisorConfig{
HypervisorPath: defaultHypervisorPath,
JailerPath: defaultJailerPath,
KernelPath: defaultKernelPath,
ImagePath: defaultImagePath,
InitrdPath: defaultInitrdPath,
RootfsType: defaultRootfsType,
FirmwarePath: defaultFirmwarePath,
FirmwareVolumePath: defaultFirmwareVolumePath,
MachineAccelerators: defaultMachineAccelerators,
CPUFeatures: defaultCPUFeatures,
HypervisorMachineType: defaultMachineType,
NumVCPUs: defaultVCPUCount,
DefaultMaxVCPUs: defaultMaxVCPUCount,
MemorySize: defaultMemSize,
MemOffset: defaultMemOffset,
VirtioMem: defaultVirtioMem,
DisableBlockDeviceUse: defaultDisableBlockDeviceUse,
DefaultBridges: defaultBridgesCount,
MemPrealloc: defaultEnableMemPrealloc,
HugePages: defaultEnableHugePages,
IOMMU: defaultEnableIOMMU,
IOMMUPlatform: defaultEnableIOMMUPlatform,
FileBackedMemRootDir: defaultFileBackedMemRootDir,
Debug: defaultEnableDebug,
DisableNestingChecks: defaultDisableNestingChecks,
BlockDeviceDriver: defaultBlockDeviceDriver,
BlockDeviceAIO: defaultBlockDeviceAIO,
BlockDeviceCacheSet: defaultBlockDeviceCacheSet,
BlockDeviceCacheDirect: defaultBlockDeviceCacheDirect,
BlockDeviceCacheNoflush: defaultBlockDeviceCacheNoflush,
EnableIOThreads: defaultEnableIOThreads,
Msize9p: defaultMsize9p,
HotplugVFIOOnRootBus: defaultHotplugVFIOOnRootBus,
PCIeRootPort: defaultPCIeRootPort,
GuestHookPath: defaultGuestHookPath,
VhostUserStorePath: defaultVhostUserStorePath,
VirtioFSCache: defaultVirtioFSCacheMode,
DisableImageNvdimm: defaultDisableImageNvdimm,
RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate,
TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate,
SGXEPCSize: defaultSGXEPCSize,
ConfidentialGuest: defaultConfidentialGuest,
SevSnpGuest: defaultSevSnpGuest,
GuestSwap: defaultGuestSwap,
Rootless: defaultRootlessHypervisor,
DisableSeccomp: defaultDisableSeccomp,
DisableGuestSeLinux: defaultDisableGuestSeLinux,
LegacySerial: defaultLegacySerial,
HypervisorPath: defaultHypervisorPath,
JailerPath: defaultJailerPath,
KernelPath: defaultKernelPath,
ImagePath: defaultImagePath,
InitrdPath: defaultInitrdPath,
RootfsType: defaultRootfsType,
FirmwarePath: defaultFirmwarePath,
FirmwareVolumePath: defaultFirmwareVolumePath,
MachineAccelerators: defaultMachineAccelerators,
CPUFeatures: defaultCPUFeatures,
HypervisorMachineType: defaultMachineType,
NumVCPUs: defaultVCPUCount,
DefaultMaxVCPUs: defaultMaxVCPUCount,
MemorySize: defaultMemSize,
MemOffset: defaultMemOffset,
VirtioMem: defaultVirtioMem,
DisableBlockDeviceUse: defaultDisableBlockDeviceUse,
DefaultBridges: defaultBridgesCount,
MemPrealloc: defaultEnableMemPrealloc,
HugePages: defaultEnableHugePages,
IOMMU: defaultEnableIOMMU,
IOMMUPlatform: defaultEnableIOMMUPlatform,
FileBackedMemRootDir: defaultFileBackedMemRootDir,
Debug: defaultEnableDebug,
DisableNestingChecks: defaultDisableNestingChecks,
BlockDeviceDriver: defaultBlockDeviceDriver,
BlockDeviceAIO: defaultBlockDeviceAIO,
BlockDeviceCacheSet: defaultBlockDeviceCacheSet,
BlockDeviceCacheDirect: defaultBlockDeviceCacheDirect,
BlockDeviceCacheNoflush: defaultBlockDeviceCacheNoflush,
EnableIOThreads: defaultEnableIOThreads,
Msize9p: defaultMsize9p,
HotplugVFIOOnRootBus: defaultHotplugVFIOOnRootBus,
PCIeRootPort: defaultPCIeRootPort,
GuestHookPath: defaultGuestHookPath,
VhostUserStorePath: defaultVhostUserStorePath,
VhostUserDeviceReconnect: defaultVhostUserDeviceReconnect,
VirtioFSCache: defaultVirtioFSCacheMode,
DisableImageNvdimm: defaultDisableImageNvdimm,
RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate,
TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate,
SGXEPCSize: defaultSGXEPCSize,
ConfidentialGuest: defaultConfidentialGuest,
SevSnpGuest: defaultSevSnpGuest,
GuestSwap: defaultGuestSwap,
Rootless: defaultRootlessHypervisor,
DisableSeccomp: defaultDisableSeccomp,
DisableGuestSeLinux: defaultDisableGuestSeLinux,
LegacySerial: defaultLegacySerial,
}
}

6
src/runtime/pkg/oci/utils.go
View File

@@ -478,6 +478,12 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig,
return err
}
if err := newAnnotationConfiguration(ocispec, vcAnnotations.VhostUserDeviceReconnect).setUint(func(reconnect uint64) {
config.HypervisorConfig.VhostUserDeviceReconnect = uint32(reconnect)
}); err != nil {
return err
}
if value, ok := ocispec.Annotations[vcAnnotations.GuestHookPath]; ok {
if value != "" {
config.HypervisorConfig.GuestHookPath = value