From 354cd3b9b6e62c35be902b926fb900c32c6bb604 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Tue, 22 Mar 2022 19:29:10 -0700 Subject: [PATCH 01/15] runtime: Base64 encode the direct volume mountInfo path This is to avoid accidentally deleting multiple volumes. Fixes #4020 Signed-off-by: Feng Wang --- src/runtime/cmd/kata-runtime/kata-volume.go | 18 ++++++++---- src/runtime/pkg/direct-volume/utils.go | 32 ++++++++++----------- src/runtime/pkg/direct-volume/utils_test.go | 11 +++---- src/runtime/virtcontainers/kata_agent.go | 6 ++-- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/runtime/cmd/kata-runtime/kata-volume.go b/src/runtime/cmd/kata-runtime/kata-volume.go index 147709129..e08e9482f 100644 --- a/src/runtime/cmd/kata-runtime/kata-volume.go +++ b/src/runtime/cmd/kata-runtime/kata-volume.go @@ -54,7 +54,10 @@ var addCommand = cli.Command{ }, }, Action: func(c *cli.Context) error { - return volume.Add(volumePath, mountInfo) + if err := volume.Add(volumePath, mountInfo); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return nil }, } @@ -69,7 +72,10 @@ var removeCommand = cli.Command{ }, }, Action: func(c *cli.Context) error { - return volume.Remove(volumePath) + if err := volume.Remove(volumePath); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return nil }, } @@ -86,9 +92,8 @@ var statsCommand = cli.Command{ Action: func(c *cli.Context) (string, error) { stats, err := Stats(volumePath) if err != nil { - return "", err + return "", cli.NewExitError(err.Error(), 1) } - return string(stats), nil }, } @@ -109,7 +114,10 @@ var resizeCommand = cli.Command{ }, }, Action: func(c *cli.Context) error { - return Resize(volumePath, size) + if err := Resize(volumePath, size); err != nil { + return cli.NewExitError(err.Error(), 1) + } + return nil }, } diff --git a/src/runtime/pkg/direct-volume/utils.go b/src/runtime/pkg/direct-volume/utils.go index 275b0508f..124e8b4ab 100644 --- a/src/runtime/pkg/direct-volume/utils.go +++ b/src/runtime/pkg/direct-volume/utils.go @@ -6,13 +6,13 @@ package volume import ( + b64 "encoding/base64" "encoding/json" "errors" "fmt" "io/ioutil" "os" "path/filepath" - "strings" ) const ( @@ -37,19 +37,20 @@ type MountInfo struct { // Add writes the mount info of a direct volume into a filesystem path known to Kata Container. func Add(volumePath string, mountInfo string) error { - volumeDir := filepath.Join(kataDirectVolumeRootPath, volumePath) + volumeDir := filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath))) stat, err := os.Stat(volumeDir) - if err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - if stat != nil && !stat.IsDir() { - return fmt.Errorf("%s should be a directory", volumeDir) - } - if errors.Is(err, os.ErrNotExist) { + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } if err := os.MkdirAll(volumeDir, 0700); err != nil { return err } } + if stat != nil && !stat.IsDir() { + return fmt.Errorf("%s should be a directory", volumeDir) + } + var deserialized MountInfo if err := json.Unmarshal([]byte(mountInfo), &deserialized); err != nil { return err @@ -60,14 +61,12 @@ func Add(volumePath string, mountInfo string) error { // Remove deletes the direct volume path including all the files inside it. func Remove(volumePath string) error { - // Find the base of the volume path to delete the whole volume path - base := strings.SplitN(volumePath, string(os.PathSeparator), 2)[0] - return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, base)) + return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)))) } // VolumeMountInfo retrieves the mount info of a direct volume. func VolumeMountInfo(volumePath string) (*MountInfo, error) { - mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName) + mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)), mountInfoFileName) if _, err := os.Stat(mountInfoFilePath); err != nil { return nil, err } @@ -84,16 +83,17 @@ func VolumeMountInfo(volumePath string) (*MountInfo, error) { // RecordSandboxId associates a sandbox id with a direct volume. func RecordSandboxId(sandboxId string, volumePath string) error { - mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName) + encodedPath := b64.URLEncoding.EncodeToString([]byte(volumePath)) + mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, encodedPath, mountInfoFileName) if _, err := os.Stat(mountInfoFilePath); err != nil { return err } - return ioutil.WriteFile(filepath.Join(kataDirectVolumeRootPath, volumePath, sandboxId), []byte(""), 0600) + return ioutil.WriteFile(filepath.Join(kataDirectVolumeRootPath, encodedPath, sandboxId), []byte(""), 0600) } func GetSandboxIdForVolume(volumePath string) (string, error) { - files, err := ioutil.ReadDir(filepath.Join(kataDirectVolumeRootPath, volumePath)) + files, err := ioutil.ReadDir(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)))) if err != nil { return "", err } diff --git a/src/runtime/pkg/direct-volume/utils_test.go b/src/runtime/pkg/direct-volume/utils_test.go index 6ca80dab8..0fa8abb1c 100644 --- a/src/runtime/pkg/direct-volume/utils_test.go +++ b/src/runtime/pkg/direct-volume/utils_test.go @@ -6,6 +6,7 @@ package volume import ( + b64 "encoding/base64" "encoding/json" "errors" "os" @@ -22,7 +23,6 @@ func TestAdd(t *testing.T) { assert.Nil(t, err) defer os.RemoveAll(kataDirectVolumeRootPath) var volumePath = "/a/b/c" - var basePath = "a" actual := MountInfo{ VolumeType: "block", Device: "/dev/sda", @@ -42,14 +42,15 @@ func TestAdd(t *testing.T) { assert.Equal(t, expected.FsType, actual.FsType) assert.Equal(t, expected.Options, actual.Options) + _, err = os.Stat(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)))) + assert.Nil(t, err) // Remove the file err = Remove(volumePath) assert.Nil(t, err) - _, err = os.Stat(filepath.Join(kataDirectVolumeRootPath, basePath)) + _, err = os.Stat(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)))) assert.True(t, errors.Is(err, os.ErrNotExist)) - - // Test invalid mount info json - assert.Error(t, Add(volumePath, "{invalid json}")) + _, err = os.Stat(filepath.Join(kataDirectVolumeRootPath)) + assert.Nil(t, err) } func TestRecordSandboxId(t *testing.T) { diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 339b2220b..c2a8c901f 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1549,11 +1549,9 @@ func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc. // Each device will be mounted at a unique location within the VM only once. Mounting // to the container specific location is handled within the OCI spec. Let's ensure that // the storage mount point is unique for each device. This is then utilized as the source - // in the OCI spec. If multiple containers mount the same block device, it's refcounted inside + // in the OCI spec. If multiple containers mount the same block device, it's ref-counted inside // the guest by Kata agent. - filename := b64.StdEncoding.EncodeToString([]byte(vol.Source)) - // Make the base64 encoding path safe. - filename = strings.ReplaceAll(filename, "/", "_") + filename := b64.URLEncoding.EncodeToString([]byte(vol.Source)) path := filepath.Join(kataGuestSandboxStorageDir(), filename) // Update applicable OCI mount source From 6a47b82c812e1b63b1082f9976729093fe7d8c03 Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Wed, 6 Apr 2022 19:31:13 -0700 Subject: [PATCH 02/15] proto: fsGroup support for direct-assigned volume This change adds two fields to the Storage pb FSGroup which is a group id that the runtime specifies to indicate to the agent to perform a chown of the mounted volume to the specified group id after mounting is complete in the guest. FSGroupChangePolicy which is a policy to indicate whether to always perform the group id ownership change or only if the root directory group id does not match with the desired group id. These two fields will allow CSI plugins to indicate to Kata that after the block device is mounted in the guest, group id ownership change should be performed on that volume. Fixes #4018 Signed-off-by: Yibo Zhuang --- src/libs/protocols/protos/agent.proto | 16 +- src/libs/protocols/protos/types.proto | 9 + .../pkg/agent/protocols/grpc/agent.pb.go | 688 ++++++++++++------ .../pkg/agent/protocols/types.pb.go | 96 ++- 4 files changed, 564 insertions(+), 245 deletions(-) diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index d0de9cf39..13a9094a8 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -399,6 +399,17 @@ message SetGuestDateTimeRequest { int64 Usec = 2; } +// FSGroup consists of the group id and group ownership change policy +// that a volume should have its ownership changed to. +message FSGroup { + // GroupID is the ID that the group ownership of the + // files in the mounted volume will need to be changed to. + uint32 group_id = 2; + // GroupChangePolicy specifies the policy for applying group id + // ownership change on a mounted volume. + types.FSGroupChangePolicy group_change_policy = 3; +} + // Storage represents both the rootfs of the container, and any volume that // could have been defined through the Mount list of the OCI specification. message Storage { @@ -422,11 +433,14 @@ message Storage { // device, "9p" for shared filesystem, or "tmpfs" for shared /dev/shm. string fstype = 4; // Options describes the additional options that might be needed to - // mount properly the storage filesytem. + // mount properly the storage filesystem. repeated string options = 5; // MountPoint refers to the path where the storage should be mounted // inside the VM. string mount_point = 6; + // FSGroup consists of the group ID and group ownership change policy + // that the mounted volume must have its group ID changed to when specified. + FSGroup fs_group = 7; } // Device represents only the devices that could have been defined through the diff --git a/src/libs/protocols/protos/types.proto b/src/libs/protocols/protos/types.proto index f2586ef91..2ff6b7ab7 100644 --- a/src/libs/protocols/protos/types.proto +++ b/src/libs/protocols/protos/types.proto @@ -16,6 +16,15 @@ enum IPFamily { v6 = 1; } +// FSGroupChangePolicy defines the policy for applying group id ownership change on a mounted volume. +enum FSGroupChangePolicy { + // Always indicates that the volume ownership will always be changed. + Always = 0; + // OnRootMismatch indicates that the volume ownership will be changed only + // when the ownership of the root directory does not match with the expected group id for the volume. + OnRootMismatch = 1; +} + message IPAddress { IPFamily family = 1; string address = 2; diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index 437c9c817..3a3e78504 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -1988,6 +1988,52 @@ func (m *SetGuestDateTimeRequest) XXX_DiscardUnknown() { var xxx_messageInfo_SetGuestDateTimeRequest proto.InternalMessageInfo +// FSGroup consists of the group id and group ownership change policy +// that a volume should have its ownership changed to. +type FSGroup struct { + // GroupID is the ID that the group ownership of the + // files in the mounted volume will need to be changed to. + GroupId uint32 `protobuf:"varint,2,opt,name=group_id,json=groupId,proto3" json:"group_id,omitempty"` + // GroupChangePolicy specifies the policy for applying group id + // ownership change on a mounted volume. + GroupChangePolicy protocols.FSGroupChangePolicy `protobuf:"varint,3,opt,name=group_change_policy,json=groupChangePolicy,proto3,enum=types.FSGroupChangePolicy" json:"group_change_policy,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FSGroup) Reset() { *m = FSGroup{} } +func (*FSGroup) ProtoMessage() {} +func (*FSGroup) Descriptor() ([]byte, []int) { + return fileDescriptor_712ce9a559fda969, []int{47} +} +func (m *FSGroup) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FSGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FSGroup.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FSGroup) XXX_Merge(src proto.Message) { + xxx_messageInfo_FSGroup.Merge(m, src) +} +func (m *FSGroup) XXX_Size() int { + return m.Size() +} +func (m *FSGroup) XXX_DiscardUnknown() { + xxx_messageInfo_FSGroup.DiscardUnknown(m) +} + +var xxx_messageInfo_FSGroup proto.InternalMessageInfo + // Storage represents both the rootfs of the container, and any volume that // could have been defined through the Mount list of the OCI specification. type Storage struct { @@ -2011,11 +2057,14 @@ type Storage struct { // device, "9p" for shared filesystem, or "tmpfs" for shared /dev/shm. Fstype string `protobuf:"bytes,4,opt,name=fstype,proto3" json:"fstype,omitempty"` // Options describes the additional options that might be needed to - // mount properly the storage filesytem. + // mount properly the storage filesystem. Options []string `protobuf:"bytes,5,rep,name=options,proto3" json:"options,omitempty"` // MountPoint refers to the path where the storage should be mounted // inside the VM. - MountPoint string `protobuf:"bytes,6,opt,name=mount_point,json=mountPoint,proto3" json:"mount_point,omitempty"` + MountPoint string `protobuf:"bytes,6,opt,name=mount_point,json=mountPoint,proto3" json:"mount_point,omitempty"` + // FSGroup consists of the group ID and group ownership change policy + // that the mounted volume must have its group ID changed to when specified. + FsGroup *FSGroup `protobuf:"bytes,7,opt,name=fs_group,json=fsGroup,proto3" json:"fs_group,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2024,7 +2073,7 @@ type Storage struct { func (m *Storage) Reset() { *m = Storage{} } func (*Storage) ProtoMessage() {} func (*Storage) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{47} + return fileDescriptor_712ce9a559fda969, []int{48} } func (m *Storage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2095,7 +2144,7 @@ type Device struct { func (m *Device) Reset() { *m = Device{} } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{48} + return fileDescriptor_712ce9a559fda969, []int{49} } func (m *Device) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2136,7 +2185,7 @@ type StringUser struct { func (m *StringUser) Reset() { *m = StringUser{} } func (*StringUser) ProtoMessage() {} func (*StringUser) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{49} + return fileDescriptor_712ce9a559fda969, []int{50} } func (m *StringUser) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2193,7 +2242,7 @@ type CopyFileRequest struct { func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } func (*CopyFileRequest) ProtoMessage() {} func (*CopyFileRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{50} + return fileDescriptor_712ce9a559fda969, []int{51} } func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2231,7 +2280,7 @@ type GetOOMEventRequest struct { func (m *GetOOMEventRequest) Reset() { *m = GetOOMEventRequest{} } func (*GetOOMEventRequest) ProtoMessage() {} func (*GetOOMEventRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{51} + return fileDescriptor_712ce9a559fda969, []int{52} } func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2270,7 +2319,7 @@ type OOMEvent struct { func (m *OOMEvent) Reset() { *m = OOMEvent{} } func (*OOMEvent) ProtoMessage() {} func (*OOMEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{52} + return fileDescriptor_712ce9a559fda969, []int{53} } func (m *OOMEvent) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2309,7 +2358,7 @@ type AddSwapRequest struct { func (m *AddSwapRequest) Reset() { *m = AddSwapRequest{} } func (*AddSwapRequest) ProtoMessage() {} func (*AddSwapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{53} + return fileDescriptor_712ce9a559fda969, []int{54} } func (m *AddSwapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2347,7 +2396,7 @@ type GetMetricsRequest struct { func (m *GetMetricsRequest) Reset() { *m = GetMetricsRequest{} } func (*GetMetricsRequest) ProtoMessage() {} func (*GetMetricsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{54} + return fileDescriptor_712ce9a559fda969, []int{55} } func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2386,7 +2435,7 @@ type Metrics struct { func (m *Metrics) Reset() { *m = Metrics{} } func (*Metrics) ProtoMessage() {} func (*Metrics) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{55} + return fileDescriptor_712ce9a559fda969, []int{56} } func (m *Metrics) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2426,7 +2475,7 @@ type VolumeStatsRequest struct { func (m *VolumeStatsRequest) Reset() { *m = VolumeStatsRequest{} } func (*VolumeStatsRequest) ProtoMessage() {} func (*VolumeStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{56} + return fileDescriptor_712ce9a559fda969, []int{57} } func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2467,7 +2516,7 @@ type ResizeVolumeRequest struct { func (m *ResizeVolumeRequest) Reset() { *m = ResizeVolumeRequest{} } func (*ResizeVolumeRequest) ProtoMessage() {} func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{57} + return fileDescriptor_712ce9a559fda969, []int{58} } func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2546,6 +2595,7 @@ func init() { proto.RegisterType((*GuestDetailsResponse)(nil), "grpc.GuestDetailsResponse") proto.RegisterType((*MemHotplugByProbeRequest)(nil), "grpc.MemHotplugByProbeRequest") proto.RegisterType((*SetGuestDateTimeRequest)(nil), "grpc.SetGuestDateTimeRequest") + proto.RegisterType((*FSGroup)(nil), "grpc.FSGroup") proto.RegisterType((*Storage)(nil), "grpc.Storage") proto.RegisterType((*Device)(nil), "grpc.Device") proto.RegisterType((*StringUser)(nil), "grpc.StringUser") @@ -2564,198 +2614,203 @@ func init() { } var fileDescriptor_712ce9a559fda969 = []byte{ - // 3055 bytes of a gzipped FileDescriptorProto + // 3127 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1a, 0xcb, 0x72, 0x24, 0x47, 0xd1, 0xf3, 0x90, 0x66, 0x26, 0xe7, 0xa5, 0x69, 0x69, 0xb5, 0xb3, 0x63, 0x5b, 0xac, 0x7b, 0xed, - 0xf5, 0xda, 0xc6, 0x92, 0xbd, 0x76, 0xb0, 0x7e, 0x84, 0x59, 0x24, 0xad, 0x2c, 0xc9, 0xb6, 0xbc, - 0x43, 0xcb, 0xc2, 0x04, 0x04, 0x74, 0xb4, 0xba, 0x6b, 0x47, 0x65, 0x4d, 0x77, 0xb5, 0xab, 0xab, - 0xb5, 0x92, 0x89, 0x20, 0x38, 0xc1, 0x8d, 0x23, 0x37, 0x7e, 0x80, 0xe0, 0xc6, 0x91, 0x0b, 0x07, - 0x0e, 0x0e, 0x4e, 0x1c, 0x39, 0x11, 0x78, 0x3f, 0x81, 0x2f, 0x20, 0xea, 0xd5, 0x5d, 0x3d, 0x0f, - 0x19, 0x14, 0x1b, 0xc1, 0x65, 0xa2, 0x33, 0x2b, 0x2b, 0x5f, 0x55, 0x99, 0x95, 0x59, 0x35, 0x30, - 0x1c, 0x61, 0x76, 0x92, 0x1e, 0xaf, 0xfb, 0x24, 0xdc, 0x38, 0xf5, 0x98, 0xf7, 0xba, 0x4f, 0x22, - 0xe6, 0xe1, 0x08, 0xd1, 0x64, 0x0a, 0x4e, 0xa8, 0xbf, 0x31, 0xc6, 0xc7, 0xc9, 0x46, 0x4c, 0x09, - 0x23, 0x3e, 0x19, 0xab, 0xaf, 0x64, 0xc3, 0x1b, 0xa1, 0x88, 0xad, 0x0b, 0xc0, 0xaa, 0x8e, 0x68, - 0xec, 0x0f, 0x1a, 0xc4, 0xc7, 0x12, 0x31, 0x68, 0xf8, 0x89, 0xfe, 0x6c, 0xb2, 0x8b, 0x18, 0x25, - 0x0a, 0x78, 0x76, 0x44, 0xc8, 0x68, 0x8c, 0x24, 0x8f, 0xe3, 0xf4, 0xd1, 0x06, 0x0a, 0x63, 0x76, - 0x21, 0x07, 0xed, 0xdf, 0x97, 0x61, 0x75, 0x9b, 0x22, 0x8f, 0xa1, 0x6d, 0xad, 0x80, 0x83, 0xbe, - 0x4c, 0x51, 0xc2, 0xac, 0x17, 0xa0, 0x95, 0x29, 0xe5, 0xe2, 0xa0, 0x5f, 0xba, 0x59, 0xba, 0xd3, - 0x70, 0x9a, 0x19, 0x6e, 0x3f, 0xb0, 0xae, 0x43, 0x0d, 0x9d, 0x23, 0x9f, 0x8f, 0x96, 0xc5, 0xe8, - 0x22, 0x07, 0xf7, 0x03, 0xeb, 0x4d, 0x68, 0x26, 0x8c, 0xe2, 0x68, 0xe4, 0xa6, 0x09, 0xa2, 0xfd, - 0xca, 0xcd, 0xd2, 0x9d, 0xe6, 0xdd, 0xa5, 0x75, 0xae, 0xf2, 0xfa, 0xa1, 0x18, 0x38, 0x4a, 0x10, - 0x75, 0x20, 0xc9, 0xbe, 0xad, 0xdb, 0x50, 0x0b, 0xd0, 0x19, 0xf6, 0x51, 0xd2, 0xaf, 0xde, 0xac, - 0xdc, 0x69, 0xde, 0x6d, 0x49, 0xf2, 0x07, 0x02, 0xe9, 0xe8, 0x41, 0xeb, 0x15, 0xa8, 0x27, 0x8c, - 0x50, 0x6f, 0x84, 0x92, 0xfe, 0x82, 0x20, 0x6c, 0x6b, 0xbe, 0x02, 0xeb, 0x64, 0xc3, 0xd6, 0x73, - 0x50, 0x79, 0xb8, 0xbd, 0xdf, 0x5f, 0x14, 0xd2, 0x41, 0x51, 0xc5, 0xc8, 0x77, 0x38, 0xda, 0xba, - 0x05, 0xed, 0xc4, 0x8b, 0x82, 0x63, 0x72, 0xee, 0xc6, 0x38, 0x88, 0x92, 0x7e, 0xed, 0x66, 0xe9, - 0x4e, 0xdd, 0x69, 0x29, 0xe4, 0x90, 0xe3, 0xec, 0xf7, 0xe0, 0xda, 0x21, 0xf3, 0x28, 0xbb, 0x82, - 0x77, 0xec, 0x23, 0x58, 0x75, 0x50, 0x48, 0xce, 0xae, 0xe4, 0xda, 0x3e, 0xd4, 0x18, 0x0e, 0x11, - 0x49, 0x99, 0x70, 0x6d, 0xdb, 0xd1, 0xa0, 0xfd, 0xc7, 0x12, 0x58, 0x3b, 0xe7, 0xc8, 0x1f, 0x52, - 0xe2, 0xa3, 0x24, 0xf9, 0x3f, 0x2d, 0xd7, 0xcb, 0x50, 0x8b, 0xa5, 0x02, 0xfd, 0xaa, 0x20, 0x57, - 0xab, 0xa0, 0xb5, 0xd2, 0xa3, 0xf6, 0x17, 0xb0, 0x72, 0x88, 0x47, 0x91, 0x37, 0x7e, 0x8a, 0xfa, - 0xae, 0xc2, 0x62, 0x22, 0x78, 0x0a, 0x55, 0xdb, 0x8e, 0x82, 0xec, 0x21, 0x58, 0x9f, 0x7b, 0x98, - 0x3d, 0x3d, 0x49, 0xf6, 0xeb, 0xb0, 0x5c, 0xe0, 0x98, 0xc4, 0x24, 0x4a, 0x90, 0x50, 0x80, 0x79, - 0x2c, 0x4d, 0x04, 0xb3, 0x05, 0x47, 0x41, 0x36, 0x81, 0xd5, 0xa3, 0x38, 0xb8, 0x62, 0x34, 0xdd, - 0x85, 0x06, 0x45, 0x09, 0x49, 0x29, 0x8f, 0x81, 0xb2, 0x70, 0xea, 0x8a, 0x74, 0xea, 0x27, 0x38, - 0x4a, 0xcf, 0x1d, 0x3d, 0xe6, 0xe4, 0x64, 0x6a, 0x7f, 0xb2, 0xe4, 0x2a, 0xfb, 0xf3, 0x3d, 0xb8, - 0x36, 0xf4, 0xd2, 0xe4, 0x2a, 0xba, 0xda, 0xef, 0xf3, 0xbd, 0x9d, 0xa4, 0xe1, 0x95, 0x26, 0xff, - 0xa1, 0x04, 0xf5, 0xed, 0x38, 0x3d, 0x4a, 0xbc, 0x11, 0xb2, 0xbe, 0x03, 0x4d, 0x46, 0x98, 0x37, - 0x76, 0x53, 0x0e, 0x0a, 0xf2, 0xaa, 0x03, 0x02, 0x25, 0x09, 0x5e, 0x80, 0x56, 0x8c, 0xa8, 0x1f, - 0xa7, 0x8a, 0xa2, 0x7c, 0xb3, 0x72, 0xa7, 0xea, 0x34, 0x25, 0x4e, 0x92, 0xac, 0xc3, 0xb2, 0x18, - 0x73, 0x71, 0xe4, 0x9e, 0x22, 0x1a, 0xa1, 0x71, 0x48, 0x02, 0x24, 0x36, 0x47, 0xd5, 0xe9, 0x89, - 0xa1, 0xfd, 0xe8, 0xe3, 0x6c, 0xc0, 0x7a, 0x15, 0x7a, 0x19, 0x3d, 0xdf, 0xf1, 0x82, 0xba, 0x2a, - 0xa8, 0xbb, 0x8a, 0xfa, 0x48, 0xa1, 0xed, 0x5f, 0x42, 0xe7, 0xb3, 0x13, 0x4a, 0x18, 0x1b, 0xe3, - 0x68, 0xf4, 0xc0, 0x63, 0x1e, 0x0f, 0xcd, 0x18, 0x51, 0x4c, 0x82, 0x44, 0x69, 0xab, 0x41, 0xeb, - 0x35, 0xe8, 0x31, 0x49, 0x8b, 0x02, 0x57, 0xd3, 0x94, 0x05, 0xcd, 0x52, 0x36, 0x30, 0x54, 0xc4, - 0x2f, 0x41, 0x27, 0x27, 0xe6, 0xc1, 0xad, 0xf4, 0x6d, 0x67, 0xd8, 0xcf, 0x70, 0x88, 0xec, 0x33, - 0xe1, 0x2b, 0xb1, 0xc8, 0xd6, 0x6b, 0xd0, 0xc8, 0xfd, 0x50, 0x12, 0x3b, 0xa4, 0x23, 0x77, 0x88, - 0x76, 0xa7, 0x53, 0xcf, 0x9c, 0xf2, 0x01, 0x74, 0x59, 0xa6, 0xb8, 0x1b, 0x78, 0xcc, 0x2b, 0x6e, - 0xaa, 0xa2, 0x55, 0x4e, 0x87, 0x15, 0x60, 0xfb, 0x7d, 0x68, 0x0c, 0x71, 0x90, 0x48, 0xc1, 0x7d, - 0xa8, 0xf9, 0x29, 0xa5, 0x28, 0x62, 0xda, 0x64, 0x05, 0x5a, 0x2b, 0xb0, 0x30, 0xc6, 0x21, 0x66, - 0xca, 0x4c, 0x09, 0xd8, 0x04, 0xe0, 0x00, 0x85, 0x84, 0x5e, 0x08, 0x87, 0xad, 0xc0, 0x82, 0xb9, - 0xb8, 0x12, 0xb0, 0x9e, 0x85, 0x46, 0xe8, 0x9d, 0x67, 0x8b, 0xca, 0x47, 0xea, 0xa1, 0x77, 0x2e, - 0x95, 0xef, 0x43, 0xed, 0x91, 0x87, 0xc7, 0x7e, 0xc4, 0x94, 0x57, 0x34, 0x98, 0x0b, 0xac, 0x9a, - 0x02, 0xff, 0x5a, 0x86, 0xa6, 0x94, 0x28, 0x15, 0x5e, 0x81, 0x05, 0xdf, 0xf3, 0x4f, 0x32, 0x91, - 0x02, 0xb0, 0x6e, 0x6b, 0x45, 0xca, 0x66, 0x86, 0xcb, 0x35, 0xd5, 0xaa, 0x6d, 0x00, 0x24, 0x8f, - 0xbd, 0x58, 0xe9, 0x56, 0x99, 0x43, 0xdc, 0xe0, 0x34, 0x52, 0xdd, 0xb7, 0xa0, 0x25, 0xf7, 0x9d, - 0x9a, 0x52, 0x9d, 0x33, 0xa5, 0x29, 0xa9, 0xe4, 0xa4, 0x5b, 0xd0, 0x4e, 0x13, 0xe4, 0x9e, 0x60, - 0x44, 0x3d, 0xea, 0x9f, 0x5c, 0xf4, 0x17, 0xe4, 0x01, 0x94, 0x26, 0x68, 0x4f, 0xe3, 0xac, 0xbb, - 0xb0, 0xc0, 0x73, 0x4b, 0xd2, 0x5f, 0x14, 0x67, 0xdd, 0x73, 0x26, 0x4b, 0x61, 0xea, 0xba, 0xf8, - 0xdd, 0x89, 0x18, 0xbd, 0x70, 0x24, 0xe9, 0xe0, 0x1d, 0x80, 0x1c, 0x69, 0x2d, 0x41, 0xe5, 0x14, - 0x5d, 0xa8, 0x38, 0xe4, 0x9f, 0xdc, 0x39, 0x67, 0xde, 0x38, 0xd5, 0x5e, 0x97, 0xc0, 0x7b, 0xe5, - 0x77, 0x4a, 0xb6, 0x0f, 0xdd, 0xad, 0xf1, 0x29, 0x26, 0xc6, 0xf4, 0x15, 0x58, 0x08, 0xbd, 0x2f, - 0x08, 0xd5, 0x9e, 0x14, 0x80, 0xc0, 0xe2, 0x88, 0x50, 0xcd, 0x42, 0x00, 0x56, 0x07, 0xca, 0x24, - 0x16, 0xfe, 0x6a, 0x38, 0x65, 0x12, 0xe7, 0x82, 0xaa, 0x86, 0x20, 0xfb, 0x9f, 0x55, 0x80, 0x5c, - 0x8a, 0xe5, 0xc0, 0x00, 0x13, 0x37, 0x41, 0x94, 0x9f, 0xef, 0xee, 0xf1, 0x05, 0x43, 0x89, 0x4b, - 0x91, 0x9f, 0xd2, 0x04, 0x9f, 0xf1, 0xf5, 0xe3, 0x66, 0x5f, 0x93, 0x66, 0x4f, 0xe8, 0xe6, 0x5c, - 0xc7, 0xe4, 0x50, 0xce, 0xdb, 0xe2, 0xd3, 0x1c, 0x3d, 0xcb, 0xda, 0x87, 0x6b, 0x39, 0xcf, 0xc0, - 0x60, 0x57, 0xbe, 0x8c, 0xdd, 0x72, 0xc6, 0x2e, 0xc8, 0x59, 0xed, 0xc0, 0x32, 0x26, 0xee, 0x97, - 0x29, 0x4a, 0x0b, 0x8c, 0x2a, 0x97, 0x31, 0xea, 0x61, 0xf2, 0x43, 0x31, 0x21, 0x67, 0x33, 0x84, - 0x1b, 0x86, 0x95, 0x3c, 0xdc, 0x0d, 0x66, 0xd5, 0xcb, 0x98, 0xad, 0x66, 0x5a, 0xf1, 0x7c, 0x90, - 0x73, 0xfc, 0x08, 0x56, 0x31, 0x71, 0x1f, 0x7b, 0x98, 0x4d, 0xb2, 0x5b, 0xf8, 0x16, 0x23, 0xf9, - 0x89, 0x56, 0xe4, 0x25, 0x8d, 0x0c, 0x11, 0x1d, 0x15, 0x8c, 0x5c, 0xfc, 0x16, 0x23, 0x0f, 0xc4, - 0x84, 0x9c, 0xcd, 0x26, 0xf4, 0x30, 0x99, 0xd4, 0xa6, 0x76, 0x19, 0x93, 0x2e, 0x26, 0x45, 0x4d, - 0xb6, 0xa0, 0x97, 0x20, 0x9f, 0x11, 0x6a, 0x6e, 0x82, 0xfa, 0x65, 0x2c, 0x96, 0x14, 0x7d, 0xc6, - 0xc3, 0xfe, 0x29, 0xb4, 0xf6, 0xd2, 0x11, 0x62, 0xe3, 0xe3, 0x2c, 0x19, 0x3c, 0xb5, 0xfc, 0x63, - 0xff, 0xbb, 0x0c, 0xcd, 0xed, 0x11, 0x25, 0x69, 0x5c, 0xc8, 0xc9, 0x32, 0x48, 0x27, 0x73, 0xb2, - 0x20, 0x11, 0x39, 0x59, 0x12, 0xbf, 0x0d, 0xad, 0x50, 0x84, 0xae, 0xa2, 0x97, 0x79, 0xa8, 0x37, - 0x15, 0xd4, 0x4e, 0x33, 0x34, 0x92, 0xd9, 0x3a, 0x40, 0x8c, 0x83, 0x44, 0xcd, 0x91, 0xe9, 0xa8, - 0xab, 0xca, 0x2d, 0x9d, 0xa2, 0x9d, 0x46, 0x9c, 0x65, 0xeb, 0x37, 0xa1, 0x79, 0xcc, 0x9d, 0xa4, - 0x26, 0x14, 0x92, 0x51, 0xee, 0x3d, 0x07, 0x8e, 0xf3, 0x20, 0xdc, 0x83, 0xf6, 0x89, 0x74, 0x99, - 0x9a, 0x24, 0xf7, 0xd0, 0x2d, 0x65, 0x49, 0x6e, 0xef, 0xba, 0xe9, 0x59, 0xb9, 0x00, 0xad, 0x13, - 0x03, 0x35, 0x38, 0x84, 0xde, 0x14, 0xc9, 0x8c, 0x1c, 0x74, 0xc7, 0xcc, 0x41, 0xcd, 0xbb, 0x96, - 0x14, 0x64, 0xce, 0x34, 0xf3, 0xd2, 0x6f, 0xcb, 0xd0, 0xfa, 0x14, 0xb1, 0xc7, 0x84, 0x9e, 0x4a, - 0x7d, 0x2d, 0xa8, 0x46, 0x5e, 0x88, 0x14, 0x47, 0xf1, 0x6d, 0xdd, 0x80, 0x3a, 0x3d, 0x97, 0x09, - 0x44, 0xad, 0x67, 0x8d, 0x9e, 0x8b, 0xc4, 0x60, 0x3d, 0x0f, 0x40, 0xcf, 0xdd, 0xd8, 0xf3, 0x4f, - 0x91, 0xf2, 0x60, 0xd5, 0x69, 0xd0, 0xf3, 0xa1, 0x44, 0xf0, 0xad, 0x40, 0xcf, 0x5d, 0x44, 0x29, - 0xa1, 0x89, 0xca, 0x55, 0x75, 0x7a, 0xbe, 0x23, 0x60, 0x35, 0x37, 0xa0, 0x24, 0x8e, 0x51, 0x20, - 0x72, 0xb4, 0x98, 0xfb, 0x40, 0x22, 0xb8, 0x54, 0xa6, 0xa5, 0x2e, 0x4a, 0xa9, 0x2c, 0x97, 0xca, - 0x72, 0xa9, 0x35, 0x39, 0x93, 0x99, 0x52, 0x59, 0x26, 0xb5, 0x2e, 0xa5, 0x32, 0x43, 0x2a, 0xcb, - 0xa5, 0x36, 0xf4, 0x5c, 0x25, 0xd5, 0xfe, 0x4d, 0x09, 0x56, 0x27, 0x0b, 0x3f, 0x55, 0x9b, 0xbe, - 0x0d, 0x2d, 0x5f, 0xac, 0x57, 0x61, 0x4f, 0xf6, 0xa6, 0x56, 0xd2, 0x69, 0xfa, 0xc6, 0x36, 0xbe, - 0x07, 0xed, 0x48, 0x3a, 0x38, 0xdb, 0x9a, 0x95, 0x7c, 0x5d, 0x4c, 0xdf, 0x3b, 0xad, 0xc8, 0x80, - 0xec, 0x00, 0xac, 0xcf, 0x29, 0x66, 0xe8, 0x90, 0x51, 0xe4, 0x85, 0x4f, 0xa3, 0xba, 0xb7, 0xa0, - 0x2a, 0xaa, 0x15, 0xbe, 0x4c, 0x2d, 0x47, 0x7c, 0xdb, 0x2f, 0xc3, 0x72, 0x41, 0x8a, 0xb2, 0x75, - 0x09, 0x2a, 0x63, 0x14, 0x09, 0xee, 0x6d, 0x87, 0x7f, 0xda, 0x1e, 0xf4, 0x1c, 0xe4, 0x05, 0x4f, - 0x4f, 0x1b, 0x25, 0xa2, 0x92, 0x8b, 0xb8, 0x03, 0x96, 0x29, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x19, - 0x5a, 0x3f, 0x84, 0xde, 0xf6, 0x98, 0x24, 0xe8, 0x90, 0x05, 0x38, 0x7a, 0x1a, 0xed, 0xc8, 0x2f, - 0x60, 0xf9, 0x33, 0x76, 0xf1, 0x39, 0x67, 0x96, 0xe0, 0xaf, 0xd0, 0x53, 0xb2, 0x8f, 0x92, 0xc7, - 0xda, 0x3e, 0x4a, 0x1e, 0xf3, 0xe6, 0xc6, 0x27, 0xe3, 0x34, 0x8c, 0x44, 0x28, 0xb4, 0x1d, 0x05, - 0xd9, 0x5b, 0xd0, 0x92, 0x35, 0xf4, 0x01, 0x09, 0xd2, 0x31, 0x9a, 0x19, 0x83, 0x6b, 0x00, 0xb1, - 0x47, 0xbd, 0x10, 0x31, 0x44, 0xe5, 0x1e, 0x6a, 0x38, 0x06, 0xc6, 0xfe, 0x5d, 0x19, 0x56, 0xe4, - 0x7d, 0xc3, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x03, 0xa8, 0x9f, 0x90, 0x84, 0x19, 0x0c, 0x33, 0x98, - 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0x16, 0x2e, 0x01, 0x2a, 0x97, 0x5f, 0x02, 0x4c, 0xb5, - 0xf9, 0xd5, 0xe9, 0x36, 0x9f, 0x47, 0x9b, 0x26, 0xc2, 0x32, 0xc6, 0x1b, 0x4e, 0x43, 0x61, 0xf6, - 0x03, 0xeb, 0x36, 0x74, 0x47, 0x5c, 0x4b, 0xf7, 0x84, 0x90, 0x53, 0x37, 0xf6, 0xd8, 0x89, 0x08, - 0xf5, 0x86, 0xd3, 0x16, 0xe8, 0x3d, 0x42, 0x4e, 0x87, 0x1e, 0x3b, 0xb1, 0xde, 0x85, 0x8e, 0x2a, - 0x03, 0x43, 0xe1, 0xa2, 0x44, 0x1d, 0x7e, 0x2a, 0x8a, 0x4c, 0xef, 0x39, 0xed, 0x53, 0x03, 0x4a, - 0xec, 0xeb, 0x70, 0xed, 0x01, 0x4a, 0x18, 0x25, 0x17, 0x45, 0xc7, 0xd8, 0xdf, 0x07, 0xd8, 0x8f, - 0x18, 0xa2, 0x8f, 0x3c, 0x1f, 0x25, 0xd6, 0x1b, 0x26, 0xa4, 0x8a, 0xa3, 0xa5, 0x75, 0x79, 0xdd, - 0x93, 0x0d, 0x38, 0x06, 0x8d, 0xbd, 0x0e, 0x8b, 0x0e, 0x49, 0x79, 0x3a, 0x7a, 0x51, 0x7f, 0xa9, - 0x79, 0x2d, 0x35, 0x4f, 0x20, 0x1d, 0x35, 0x66, 0xef, 0xe9, 0x16, 0x36, 0x67, 0xa7, 0x96, 0x68, - 0x1d, 0x1a, 0x58, 0xe3, 0x54, 0x56, 0x99, 0x16, 0x9d, 0x93, 0xd8, 0xef, 0xc3, 0xb2, 0xe4, 0x24, - 0x39, 0x6b, 0x36, 0x2f, 0xc2, 0x22, 0xd5, 0x6a, 0x94, 0xf2, 0x7b, 0x1e, 0x45, 0xa4, 0xc6, 0xb8, - 0x3f, 0x3e, 0xc1, 0x09, 0xcb, 0x0d, 0xd1, 0xfe, 0x58, 0x86, 0x1e, 0x1f, 0x28, 0xf0, 0xb4, 0x3f, - 0x84, 0xd6, 0xa6, 0x33, 0xfc, 0x14, 0xe1, 0xd1, 0xc9, 0x31, 0xcf, 0x9e, 0xdf, 0x2b, 0xc2, 0xca, - 0x60, 0x4b, 0x69, 0x6b, 0x0c, 0x39, 0x05, 0x3a, 0xfb, 0x23, 0x58, 0xdd, 0x0c, 0x02, 0x13, 0xa5, - 0xb5, 0x7e, 0x03, 0x1a, 0x91, 0xc1, 0xce, 0x38, 0xb3, 0x0a, 0xd4, 0x39, 0x91, 0xfd, 0x33, 0x58, - 0x7e, 0x18, 0x8d, 0x71, 0x84, 0xb6, 0x87, 0x47, 0x07, 0x28, 0xcb, 0x45, 0x16, 0x54, 0x79, 0xcd, - 0x26, 0x78, 0xd4, 0x1d, 0xf1, 0xcd, 0x83, 0x33, 0x3a, 0x76, 0xfd, 0x38, 0x4d, 0xd4, 0x65, 0xcf, - 0x62, 0x74, 0xbc, 0x1d, 0xa7, 0x09, 0x3f, 0x5c, 0x78, 0x71, 0x41, 0xa2, 0xf1, 0x85, 0x88, 0xd0, - 0xba, 0x53, 0xf3, 0xe3, 0xf4, 0x61, 0x34, 0xbe, 0xb0, 0xbf, 0x2b, 0x3a, 0x70, 0x84, 0x02, 0xc7, - 0x8b, 0x02, 0x12, 0x3e, 0x40, 0x67, 0x86, 0x84, 0xac, 0xdb, 0xd3, 0x99, 0xe8, 0xeb, 0x12, 0xb4, - 0x36, 0x47, 0x28, 0x62, 0x0f, 0x10, 0xf3, 0xf0, 0x58, 0x74, 0x74, 0x67, 0x88, 0x26, 0x98, 0x44, - 0x2a, 0xdc, 0x34, 0xc8, 0x1b, 0x72, 0x1c, 0x61, 0xe6, 0x06, 0x1e, 0x0a, 0x49, 0x24, 0xb8, 0xd4, - 0x1d, 0xe0, 0xa8, 0x07, 0x02, 0x63, 0xbd, 0x0c, 0x5d, 0x79, 0x19, 0xe7, 0x9e, 0x78, 0x51, 0x30, - 0xe6, 0x81, 0x5e, 0x11, 0xa1, 0xd9, 0x91, 0xe8, 0x3d, 0x85, 0xb5, 0x5e, 0x81, 0x25, 0x15, 0x86, - 0x39, 0x65, 0x55, 0x50, 0x76, 0x15, 0xbe, 0x40, 0x9a, 0xc6, 0x31, 0xa1, 0x2c, 0x71, 0x13, 0xe4, - 0xfb, 0x24, 0x8c, 0x55, 0x3b, 0xd4, 0xd5, 0xf8, 0x43, 0x89, 0xb6, 0x47, 0xb0, 0xbc, 0xcb, 0xed, - 0x54, 0x96, 0xe4, 0xdb, 0xaa, 0x13, 0xa2, 0xd0, 0x3d, 0x1e, 0x13, 0xff, 0xd4, 0xe5, 0xc9, 0x51, - 0x79, 0x98, 0x17, 0x5c, 0x5b, 0x1c, 0x79, 0x88, 0xbf, 0x12, 0x9d, 0x3f, 0xa7, 0x3a, 0x21, 0x2c, - 0x1e, 0xa7, 0x23, 0x37, 0xa6, 0xe4, 0x18, 0x29, 0x13, 0xbb, 0x21, 0x0a, 0xf7, 0x24, 0x7e, 0xc8, - 0xd1, 0xf6, 0x9f, 0x4b, 0xb0, 0x52, 0x94, 0xa4, 0x52, 0xfd, 0x06, 0xac, 0x14, 0x45, 0xa9, 0xe3, - 0x5f, 0x96, 0x97, 0x3d, 0x53, 0xa0, 0x2c, 0x04, 0xee, 0x41, 0x5b, 0x5c, 0xdd, 0xba, 0x81, 0xe4, - 0x54, 0x2c, 0x7a, 0xcc, 0x75, 0x71, 0x5a, 0x9e, 0xb9, 0x4a, 0xef, 0xc2, 0x0d, 0x65, 0xbe, 0x3b, - 0xad, 0xb6, 0xdc, 0x10, 0xab, 0x8a, 0xe0, 0x60, 0x42, 0xfb, 0x4f, 0xa0, 0x9f, 0xa3, 0xb6, 0x2e, - 0x04, 0x32, 0xdf, 0xcc, 0xcb, 0x13, 0xc6, 0x6e, 0x06, 0x01, 0x15, 0x51, 0x52, 0x75, 0x66, 0x0d, - 0xd9, 0xf7, 0xe1, 0xfa, 0x21, 0x62, 0xd2, 0x1b, 0x1e, 0x53, 0x9d, 0x88, 0x64, 0xb6, 0x04, 0x95, - 0x43, 0xe4, 0x0b, 0xe3, 0x2b, 0x0e, 0xff, 0xe4, 0x1b, 0xf0, 0x28, 0x41, 0xbe, 0xb0, 0xb2, 0xe2, - 0x88, 0x6f, 0xfb, 0x4f, 0x25, 0xa8, 0xa9, 0xe4, 0xcc, 0x0f, 0x98, 0x80, 0xe2, 0x33, 0x44, 0xd5, - 0xd6, 0x53, 0x90, 0xf5, 0x12, 0x74, 0xe4, 0x97, 0x4b, 0x62, 0x86, 0x49, 0x96, 0xf2, 0xdb, 0x12, - 0xfb, 0x50, 0x22, 0xc5, 0xe5, 0x9b, 0xb8, 0xfe, 0x52, 0x9d, 0xa6, 0x82, 0x38, 0xfe, 0x51, 0xc2, - 0x23, 0x5c, 0xa4, 0xf8, 0x86, 0xa3, 0x20, 0xbe, 0xd5, 0x35, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c, - 0xab, 0x87, 0x24, 0x8d, 0x98, 0x1b, 0x13, 0x1c, 0x31, 0x95, 0xd3, 0x41, 0xa0, 0x86, 0x1c, 0x63, - 0xff, 0xba, 0x04, 0x8b, 0xf2, 0x02, 0x9a, 0xf7, 0xb6, 0xd9, 0xc9, 0x5a, 0xc6, 0xa2, 0x4a, 0x11, - 0xb2, 0xe4, 0x69, 0x2a, 0xbe, 0x79, 0x1c, 0x9f, 0x85, 0xf2, 0x7c, 0x50, 0xaa, 0x9d, 0x85, 0xe2, - 0x60, 0x78, 0x09, 0x3a, 0xf9, 0x01, 0x2d, 0xc6, 0xa5, 0x8a, 0xed, 0x0c, 0x2b, 0xc8, 0xe6, 0x6a, - 0x6a, 0xff, 0x98, 0xb7, 0xf4, 0xd9, 0xe5, 0xeb, 0x12, 0x54, 0xd2, 0x4c, 0x19, 0xfe, 0xc9, 0x31, - 0xa3, 0xec, 0x68, 0xe7, 0x9f, 0xd6, 0x6d, 0xe8, 0x78, 0x41, 0x80, 0xf9, 0x74, 0x6f, 0xbc, 0x8b, - 0x83, 0x2c, 0x48, 0x8b, 0x58, 0xfb, 0x6f, 0x25, 0xe8, 0x6e, 0x93, 0xf8, 0xe2, 0x43, 0x3c, 0x46, - 0x46, 0x06, 0x11, 0x4a, 0xaa, 0x93, 0x9d, 0x7f, 0xf3, 0x6a, 0xf5, 0x11, 0x1e, 0x23, 0x19, 0x5a, - 0x72, 0x65, 0xeb, 0x1c, 0x21, 0xc2, 0x4a, 0x0f, 0x66, 0xd7, 0x6e, 0x6d, 0x39, 0x78, 0x40, 0x02, - 0x51, 0x97, 0x07, 0x98, 0xba, 0xd9, 0x25, 0x5b, 0xdb, 0xa9, 0x05, 0x98, 0x8a, 0x21, 0x65, 0xc8, - 0x82, 0xb8, 0x44, 0x35, 0x0d, 0x59, 0x94, 0x18, 0x6e, 0xc8, 0x2a, 0x2c, 0x92, 0x47, 0x8f, 0x12, - 0xc4, 0x44, 0x05, 0x5d, 0x71, 0x14, 0x94, 0xa5, 0xb9, 0xba, 0x91, 0xe6, 0x56, 0xc0, 0xda, 0x45, - 0xec, 0xe1, 0xc3, 0x83, 0x9d, 0x33, 0x14, 0x31, 0x7d, 0x3a, 0xbc, 0x0e, 0x75, 0x8d, 0xfa, 0x6f, - 0xae, 0x27, 0x5f, 0x85, 0xce, 0x66, 0x10, 0x1c, 0x3e, 0xf6, 0x62, 0xed, 0x8f, 0x3e, 0xd4, 0x86, - 0xdb, 0xfb, 0x43, 0xe9, 0x92, 0x0a, 0x37, 0x40, 0x81, 0xfc, 0x34, 0xda, 0x45, 0xec, 0x00, 0x31, - 0x8a, 0xfd, 0xec, 0x34, 0xba, 0x05, 0x35, 0x85, 0xe1, 0x33, 0x43, 0xf9, 0xa9, 0xd3, 0xac, 0x02, - 0xed, 0x1f, 0x80, 0xf5, 0x23, 0x5e, 0x57, 0x21, 0x59, 0x54, 0x2b, 0x49, 0xaf, 0x42, 0xef, 0x4c, - 0x60, 0x5d, 0x59, 0x70, 0x18, 0xcb, 0xd0, 0x95, 0x03, 0x22, 0x06, 0x85, 0xec, 0x23, 0x58, 0x96, - 0x65, 0xa0, 0xe4, 0x73, 0x05, 0x16, 0xdc, 0x87, 0xd9, 0x7a, 0x56, 0x1d, 0xf1, 0x7d, 0xf7, 0x2f, - 0x3d, 0x75, 0x54, 0xa8, 0x5b, 0x07, 0x6b, 0x17, 0xba, 0x13, 0x4f, 0x44, 0x96, 0xba, 0x86, 0x9a, - 0xfd, 0x72, 0x34, 0x58, 0x5d, 0x97, 0x4f, 0x4e, 0xeb, 0xfa, 0xc9, 0x69, 0x7d, 0x27, 0x8c, 0xd9, - 0x85, 0xb5, 0x03, 0x9d, 0xe2, 0x63, 0x8a, 0xf5, 0xac, 0xae, 0xda, 0x66, 0x3c, 0xb1, 0xcc, 0x65, - 0xb3, 0x0b, 0xdd, 0x89, 0x77, 0x15, 0xad, 0xcf, 0xec, 0xe7, 0x96, 0xb9, 0x8c, 0xee, 0x43, 0xd3, - 0x78, 0x48, 0xb1, 0xfa, 0x92, 0xc9, 0xf4, 0xdb, 0xca, 0x5c, 0x06, 0xdb, 0xd0, 0x2e, 0xbc, 0x6d, - 0x58, 0x03, 0x65, 0xcf, 0x8c, 0x07, 0x8f, 0xb9, 0x4c, 0xb6, 0xa0, 0x69, 0x3c, 0x31, 0x68, 0x2d, - 0xa6, 0xdf, 0x31, 0x06, 0x37, 0x66, 0x8c, 0xa8, 0x13, 0x69, 0x17, 0xba, 0x13, 0xef, 0x0e, 0xda, - 0x25, 0xb3, 0x9f, 0x23, 0xe6, 0x2a, 0xf3, 0xb1, 0x58, 0x22, 0xa3, 0xad, 0x34, 0x96, 0x68, 0xfa, - 0x95, 0x61, 0xf0, 0xdc, 0xec, 0x41, 0xa5, 0xd5, 0x0e, 0x74, 0x8a, 0x0f, 0x0c, 0x9a, 0xd9, 0xcc, - 0x67, 0x87, 0xcb, 0xd7, 0xbb, 0xf0, 0xd6, 0x90, 0xaf, 0xf7, 0xac, 0x27, 0x88, 0xb9, 0x8c, 0x36, - 0x01, 0x54, 0x13, 0x19, 0xe0, 0x28, 0x73, 0xf4, 0x54, 0xf3, 0x9a, 0x39, 0x7a, 0x46, 0xc3, 0x79, - 0x1f, 0x40, 0xf6, 0x7e, 0x01, 0x49, 0x99, 0x75, 0x5d, 0xab, 0x31, 0xd1, 0x70, 0x0e, 0xfa, 0xd3, - 0x03, 0x53, 0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x01, 0x40, 0xde, 0x53, 0x6a, 0x06, 0x53, 0x5d, - 0xe6, 0x25, 0x3e, 0x68, 0x99, 0x1d, 0xa4, 0xa5, 0x6c, 0x9d, 0xd1, 0x55, 0x5e, 0xc2, 0xa2, 0x3b, - 0xd1, 0x21, 0x14, 0x37, 0xdb, 0x64, 0xe3, 0x30, 0x98, 0xea, 0x12, 0xac, 0x7b, 0xd0, 0x32, 0x5b, - 0x03, 0xad, 0xc5, 0x8c, 0x76, 0x61, 0x50, 0x68, 0x0f, 0xac, 0xfb, 0xd0, 0x29, 0xb6, 0x05, 0x7a, - 0x4b, 0xcd, 0x6c, 0x16, 0x06, 0xea, 0xd2, 0xcb, 0x20, 0x7f, 0x0b, 0x20, 0x6f, 0x1f, 0xb4, 0xfb, - 0xa6, 0x1a, 0x8a, 0x09, 0xa9, 0xbb, 0xd0, 0x9d, 0x68, 0x0b, 0xb4, 0xc5, 0xb3, 0xbb, 0x85, 0xb9, - 0xae, 0x7b, 0x1b, 0x20, 0x3f, 0x2e, 0xb4, 0xf4, 0xa9, 0x03, 0x64, 0xd0, 0xd6, 0x17, 0x82, 0x92, - 0x6e, 0x1b, 0xda, 0x85, 0x9e, 0x59, 0xa7, 0x99, 0x59, 0x8d, 0xf4, 0x65, 0xc9, 0xb7, 0xd8, 0x60, - 0x6a, 0xcf, 0xcd, 0x6c, 0x3b, 0x2f, 0xdb, 0x3f, 0x66, 0x57, 0xa3, 0x57, 0x6e, 0x46, 0xa7, 0xf3, - 0x2d, 0xf1, 0x6c, 0x76, 0x2e, 0x46, 0x3c, 0xcf, 0x68, 0x68, 0xe6, 0x32, 0xda, 0x83, 0xee, 0xae, - 0x2e, 0x4a, 0x55, 0xc1, 0xac, 0xd4, 0x99, 0xd1, 0x20, 0x0c, 0x06, 0xb3, 0x86, 0x54, 0x50, 0x7d, - 0x0c, 0xbd, 0xa9, 0x62, 0xd9, 0x5a, 0xcb, 0xae, 0x65, 0x67, 0x56, 0xd1, 0x73, 0xd5, 0xda, 0x87, - 0xa5, 0xc9, 0x5a, 0xd9, 0x7a, 0x5e, 0x25, 0xca, 0xd9, 0x35, 0xf4, 0x5c, 0x56, 0xef, 0x42, 0x5d, - 0xd7, 0x66, 0x96, 0xba, 0xfe, 0x9e, 0xa8, 0xd5, 0xe6, 0x4e, 0xbd, 0x07, 0x4d, 0xa3, 0x14, 0xd2, - 0xd9, 0x6e, 0xba, 0x3a, 0x1a, 0xa8, 0xdb, 0xea, 0x8c, 0xf2, 0x1e, 0xd4, 0x54, 0xf9, 0x63, 0xad, - 0x64, 0x9b, 0xdc, 0xa8, 0x86, 0x2e, 0xdb, 0x61, 0xbb, 0x88, 0x19, 0x45, 0x8d, 0x16, 0x3a, 0x5d, - 0xe7, 0xe8, 0x14, 0x5b, 0x18, 0x51, 0x6b, 0xb1, 0x09, 0x2d, 0xb3, 0xac, 0xd1, 0x4b, 0x3a, 0xa3, - 0xd4, 0x99, 0xa7, 0xc9, 0xd6, 0xf9, 0xd7, 0xdf, 0xac, 0x3d, 0xf3, 0x8f, 0x6f, 0xd6, 0x9e, 0xf9, - 0xd5, 0x93, 0xb5, 0xd2, 0xd7, 0x4f, 0xd6, 0x4a, 0x7f, 0x7f, 0xb2, 0x56, 0xfa, 0xd7, 0x93, 0xb5, - 0xd2, 0x4f, 0x7e, 0xfe, 0x3f, 0xfe, 0x0f, 0x87, 0xa6, 0x11, 0xc3, 0x21, 0xda, 0x38, 0xc3, 0x94, - 0x19, 0x43, 0xf1, 0xe9, 0x48, 0xfe, 0x19, 0xc7, 0xf8, 0x8f, 0x0e, 0xd7, 0xf2, 0x78, 0x51, 0xc0, - 0x6f, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x5c, 0x4e, 0xa6, 0xdc, 0xf0, 0x23, 0x00, 0x00, + 0xb5, 0x6c, 0x63, 0xc9, 0x5e, 0x3b, 0x58, 0x3f, 0xc2, 0x2c, 0x92, 0x56, 0x96, 0x64, 0x5b, 0xde, + 0xa1, 0x65, 0x61, 0x02, 0x02, 0x3a, 0x7a, 0xba, 0x4b, 0x33, 0x65, 0x4d, 0x77, 0xb5, 0xab, 0xab, + 0xb5, 0x1a, 0x13, 0x41, 0x70, 0x82, 0x1b, 0x47, 0x6e, 0xfc, 0x00, 0xc1, 0x1f, 0x70, 0xe1, 0xc0, + 0xc1, 0xc1, 0x89, 0x23, 0x17, 0x08, 0xbc, 0x9f, 0xc0, 0x17, 0x10, 0xf5, 0xea, 0xc7, 0x3c, 0x64, + 0x50, 0x6c, 0x04, 0x97, 0x89, 0xce, 0xac, 0xac, 0x7c, 0x55, 0x65, 0x56, 0x66, 0xd5, 0x40, 0x7f, + 0x88, 0xd9, 0x28, 0x1e, 0x6c, 0xb9, 0xc4, 0xdf, 0x3e, 0x77, 0x98, 0xf3, 0xba, 0x4b, 0x02, 0xe6, + 0xe0, 0x00, 0xd1, 0x68, 0x06, 0x8e, 0xa8, 0xbb, 0x3d, 0xc6, 0x83, 0x68, 0x3b, 0xa4, 0x84, 0x11, + 0x97, 0x8c, 0xd5, 0x57, 0xb4, 0xed, 0x0c, 0x51, 0xc0, 0xb6, 0x04, 0x60, 0x94, 0x87, 0x34, 0x74, + 0x7b, 0x35, 0xe2, 0x62, 0x89, 0xe8, 0xd5, 0xdc, 0x48, 0x7f, 0xd6, 0xd9, 0x24, 0x44, 0x91, 0x02, + 0x9e, 0x1d, 0x12, 0x32, 0x1c, 0x23, 0xc9, 0x63, 0x10, 0x9f, 0x6d, 0x23, 0x3f, 0x64, 0x13, 0x39, + 0x68, 0xfe, 0xbe, 0x08, 0xeb, 0x7b, 0x14, 0x39, 0x0c, 0xed, 0x69, 0x05, 0x2c, 0xf4, 0x65, 0x8c, + 0x22, 0x66, 0xbc, 0x00, 0x8d, 0x44, 0x29, 0x1b, 0x7b, 0xdd, 0xc2, 0xed, 0xc2, 0x66, 0xcd, 0xaa, + 0x27, 0xb8, 0x23, 0xcf, 0xb8, 0x09, 0x15, 0x74, 0x89, 0x5c, 0x3e, 0x5a, 0x14, 0xa3, 0xcb, 0x1c, + 0x3c, 0xf2, 0x8c, 0x37, 0xa1, 0x1e, 0x31, 0x8a, 0x83, 0xa1, 0x1d, 0x47, 0x88, 0x76, 0x4b, 0xb7, + 0x0b, 0x9b, 0xf5, 0x7b, 0x2b, 0x5b, 0x5c, 0xe5, 0xad, 0x13, 0x31, 0x70, 0x1a, 0x21, 0x6a, 0x41, + 0x94, 0x7c, 0x1b, 0x77, 0xa1, 0xe2, 0xa1, 0x0b, 0xec, 0xa2, 0xa8, 0x5b, 0xbe, 0x5d, 0xda, 0xac, + 0xdf, 0x6b, 0x48, 0xf2, 0x87, 0x02, 0x69, 0xe9, 0x41, 0xe3, 0x15, 0xa8, 0x46, 0x8c, 0x50, 0x67, + 0x88, 0xa2, 0xee, 0x92, 0x20, 0x6c, 0x6a, 0xbe, 0x02, 0x6b, 0x25, 0xc3, 0xc6, 0x73, 0x50, 0x7a, + 0xb4, 0x77, 0xd4, 0x5d, 0x16, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, 0xb8, 0x03, 0xcd, + 0xc8, 0x09, 0xbc, 0x01, 0xb9, 0xb4, 0x43, 0xec, 0x05, 0x51, 0xb7, 0x72, 0xbb, 0xb0, 0x59, 0xb5, + 0x1a, 0x0a, 0xd9, 0xe7, 0x38, 0xf3, 0x3d, 0xb8, 0x71, 0xc2, 0x1c, 0xca, 0xae, 0xe1, 0x1d, 0xf3, + 0x14, 0xd6, 0x2d, 0xe4, 0x93, 0x8b, 0x6b, 0xb9, 0xb6, 0x0b, 0x15, 0x86, 0x7d, 0x44, 0x62, 0x26, + 0x5c, 0xdb, 0xb4, 0x34, 0x68, 0xfe, 0xb1, 0x00, 0xc6, 0xfe, 0x25, 0x72, 0xfb, 0x94, 0xb8, 0x28, + 0x8a, 0xfe, 0x4f, 0xcb, 0xf5, 0x32, 0x54, 0x42, 0xa9, 0x40, 0xb7, 0x2c, 0xc8, 0xd5, 0x2a, 0x68, + 0xad, 0xf4, 0xa8, 0xf9, 0x05, 0xac, 0x9d, 0xe0, 0x61, 0xe0, 0x8c, 0x9f, 0xa2, 0xbe, 0xeb, 0xb0, + 0x1c, 0x09, 0x9e, 0x42, 0xd5, 0xa6, 0xa5, 0x20, 0xb3, 0x0f, 0xc6, 0xe7, 0x0e, 0x66, 0x4f, 0x4f, + 0x92, 0xf9, 0x3a, 0xac, 0xe6, 0x38, 0x46, 0x21, 0x09, 0x22, 0x24, 0x14, 0x60, 0x0e, 0x8b, 0x23, + 0xc1, 0x6c, 0xc9, 0x52, 0x90, 0x49, 0x60, 0xfd, 0x34, 0xf4, 0xae, 0x19, 0x4d, 0xf7, 0xa0, 0x46, + 0x51, 0x44, 0x62, 0xca, 0x63, 0xa0, 0x28, 0x9c, 0xba, 0x26, 0x9d, 0xfa, 0x09, 0x0e, 0xe2, 0x4b, + 0x4b, 0x8f, 0x59, 0x29, 0x99, 0xda, 0x9f, 0x2c, 0xba, 0xce, 0xfe, 0x7c, 0x0f, 0x6e, 0xf4, 0x9d, + 0x38, 0xba, 0x8e, 0xae, 0xe6, 0xfb, 0x7c, 0x6f, 0x47, 0xb1, 0x7f, 0xad, 0xc9, 0x7f, 0x28, 0x40, + 0x75, 0x2f, 0x8c, 0x4f, 0x23, 0x67, 0x88, 0x8c, 0xef, 0x40, 0x9d, 0x11, 0xe6, 0x8c, 0xed, 0x98, + 0x83, 0x82, 0xbc, 0x6c, 0x81, 0x40, 0x49, 0x82, 0x17, 0xa0, 0x11, 0x22, 0xea, 0x86, 0xb1, 0xa2, + 0x28, 0xde, 0x2e, 0x6d, 0x96, 0xad, 0xba, 0xc4, 0x49, 0x92, 0x2d, 0x58, 0x15, 0x63, 0x36, 0x0e, + 0xec, 0x73, 0x44, 0x03, 0x34, 0xf6, 0x89, 0x87, 0xc4, 0xe6, 0x28, 0x5b, 0x1d, 0x31, 0x74, 0x14, + 0x7c, 0x9c, 0x0c, 0x18, 0xaf, 0x42, 0x27, 0xa1, 0xe7, 0x3b, 0x5e, 0x50, 0x97, 0x05, 0x75, 0x5b, + 0x51, 0x9f, 0x2a, 0xb4, 0xf9, 0x4b, 0x68, 0x7d, 0x36, 0xa2, 0x84, 0xb1, 0x31, 0x0e, 0x86, 0x0f, + 0x1d, 0xe6, 0xf0, 0xd0, 0x0c, 0x11, 0xc5, 0xc4, 0x8b, 0x94, 0xb6, 0x1a, 0x34, 0x5e, 0x83, 0x0e, + 0x93, 0xb4, 0xc8, 0xb3, 0x35, 0x4d, 0x51, 0xd0, 0xac, 0x24, 0x03, 0x7d, 0x45, 0xfc, 0x12, 0xb4, + 0x52, 0x62, 0x1e, 0xdc, 0x4a, 0xdf, 0x66, 0x82, 0xfd, 0x0c, 0xfb, 0xc8, 0xbc, 0x10, 0xbe, 0x12, + 0x8b, 0x6c, 0xbc, 0x06, 0xb5, 0xd4, 0x0f, 0x05, 0xb1, 0x43, 0x5a, 0x72, 0x87, 0x68, 0x77, 0x5a, + 0xd5, 0xc4, 0x29, 0x1f, 0x40, 0x9b, 0x25, 0x8a, 0xdb, 0x9e, 0xc3, 0x9c, 0xfc, 0xa6, 0xca, 0x5b, + 0x65, 0xb5, 0x58, 0x0e, 0x36, 0xdf, 0x87, 0x5a, 0x1f, 0x7b, 0x91, 0x14, 0xdc, 0x85, 0x8a, 0x1b, + 0x53, 0x8a, 0x02, 0xa6, 0x4d, 0x56, 0xa0, 0xb1, 0x06, 0x4b, 0x63, 0xec, 0x63, 0xa6, 0xcc, 0x94, + 0x80, 0x49, 0x00, 0x8e, 0x91, 0x4f, 0xe8, 0x44, 0x38, 0x6c, 0x0d, 0x96, 0xb2, 0x8b, 0x2b, 0x01, + 0xe3, 0x59, 0xa8, 0xf9, 0xce, 0x65, 0xb2, 0xa8, 0x7c, 0xa4, 0xea, 0x3b, 0x97, 0x52, 0xf9, 0x2e, + 0x54, 0xce, 0x1c, 0x3c, 0x76, 0x03, 0xa6, 0xbc, 0xa2, 0xc1, 0x54, 0x60, 0x39, 0x2b, 0xf0, 0x2f, + 0x45, 0xa8, 0x4b, 0x89, 0x52, 0xe1, 0x35, 0x58, 0x72, 0x1d, 0x77, 0x94, 0x88, 0x14, 0x80, 0x71, + 0x57, 0x2b, 0x52, 0xcc, 0x66, 0xb8, 0x54, 0x53, 0xad, 0xda, 0x36, 0x40, 0xf4, 0xd8, 0x09, 0x95, + 0x6e, 0xa5, 0x05, 0xc4, 0x35, 0x4e, 0x23, 0xd5, 0x7d, 0x0b, 0x1a, 0x72, 0xdf, 0xa9, 0x29, 0xe5, + 0x05, 0x53, 0xea, 0x92, 0x4a, 0x4e, 0xba, 0x03, 0xcd, 0x38, 0x42, 0xf6, 0x08, 0x23, 0xea, 0x50, + 0x77, 0x34, 0xe9, 0x2e, 0xc9, 0x03, 0x28, 0x8e, 0xd0, 0xa1, 0xc6, 0x19, 0xf7, 0x60, 0x89, 0xe7, + 0x96, 0xa8, 0xbb, 0x2c, 0xce, 0xba, 0xe7, 0xb2, 0x2c, 0x85, 0xa9, 0x5b, 0xe2, 0x77, 0x3f, 0x60, + 0x74, 0x62, 0x49, 0xd2, 0xde, 0x3b, 0x00, 0x29, 0xd2, 0x58, 0x81, 0xd2, 0x39, 0x9a, 0xa8, 0x38, + 0xe4, 0x9f, 0xdc, 0x39, 0x17, 0xce, 0x38, 0xd6, 0x5e, 0x97, 0xc0, 0x7b, 0xc5, 0x77, 0x0a, 0xa6, + 0x0b, 0xed, 0xdd, 0xf1, 0x39, 0x26, 0x99, 0xe9, 0x6b, 0xb0, 0xe4, 0x3b, 0x5f, 0x10, 0xaa, 0x3d, + 0x29, 0x00, 0x81, 0xc5, 0x01, 0xa1, 0x9a, 0x85, 0x00, 0x8c, 0x16, 0x14, 0x49, 0x28, 0xfc, 0x55, + 0xb3, 0x8a, 0x24, 0x4c, 0x05, 0x95, 0x33, 0x82, 0xcc, 0x7f, 0x96, 0x01, 0x52, 0x29, 0x86, 0x05, + 0x3d, 0x4c, 0xec, 0x08, 0x51, 0x7e, 0xbe, 0xdb, 0x83, 0x09, 0x43, 0x91, 0x4d, 0x91, 0x1b, 0xd3, + 0x08, 0x5f, 0xf0, 0xf5, 0xe3, 0x66, 0xdf, 0x90, 0x66, 0x4f, 0xe9, 0x66, 0xdd, 0xc4, 0xe4, 0x44, + 0xce, 0xdb, 0xe5, 0xd3, 0x2c, 0x3d, 0xcb, 0x38, 0x82, 0x1b, 0x29, 0x4f, 0x2f, 0xc3, 0xae, 0x78, + 0x15, 0xbb, 0xd5, 0x84, 0x9d, 0x97, 0xb2, 0xda, 0x87, 0x55, 0x4c, 0xec, 0x2f, 0x63, 0x14, 0xe7, + 0x18, 0x95, 0xae, 0x62, 0xd4, 0xc1, 0xe4, 0x87, 0x62, 0x42, 0xca, 0xa6, 0x0f, 0xb7, 0x32, 0x56, + 0xf2, 0x70, 0xcf, 0x30, 0x2b, 0x5f, 0xc5, 0x6c, 0x3d, 0xd1, 0x8a, 0xe7, 0x83, 0x94, 0xe3, 0x47, + 0xb0, 0x8e, 0x89, 0xfd, 0xd8, 0xc1, 0x6c, 0x9a, 0xdd, 0xd2, 0xb7, 0x18, 0xc9, 0x4f, 0xb4, 0x3c, + 0x2f, 0x69, 0xa4, 0x8f, 0xe8, 0x30, 0x67, 0xe4, 0xf2, 0xb7, 0x18, 0x79, 0x2c, 0x26, 0xa4, 0x6c, + 0x76, 0xa0, 0x83, 0xc9, 0xb4, 0x36, 0x95, 0xab, 0x98, 0xb4, 0x31, 0xc9, 0x6b, 0xb2, 0x0b, 0x9d, + 0x08, 0xb9, 0x8c, 0xd0, 0xec, 0x26, 0xa8, 0x5e, 0xc5, 0x62, 0x45, 0xd1, 0x27, 0x3c, 0xcc, 0x9f, + 0x42, 0xe3, 0x30, 0x1e, 0x22, 0x36, 0x1e, 0x24, 0xc9, 0xe0, 0xa9, 0xe5, 0x1f, 0xf3, 0xdf, 0x45, + 0xa8, 0xef, 0x0d, 0x29, 0x89, 0xc3, 0x5c, 0x4e, 0x96, 0x41, 0x3a, 0x9d, 0x93, 0x05, 0x89, 0xc8, + 0xc9, 0x92, 0xf8, 0x6d, 0x68, 0xf8, 0x22, 0x74, 0x15, 0xbd, 0xcc, 0x43, 0x9d, 0x99, 0xa0, 0xb6, + 0xea, 0x7e, 0x26, 0x99, 0x6d, 0x01, 0x84, 0xd8, 0x8b, 0xd4, 0x1c, 0x99, 0x8e, 0xda, 0xaa, 0xdc, + 0xd2, 0x29, 0xda, 0xaa, 0x85, 0x49, 0xb6, 0x7e, 0x13, 0xea, 0x03, 0xee, 0x24, 0x35, 0x21, 0x97, + 0x8c, 0x52, 0xef, 0x59, 0x30, 0x48, 0x83, 0xf0, 0x10, 0x9a, 0x23, 0xe9, 0x32, 0x35, 0x49, 0xee, + 0xa1, 0x3b, 0xca, 0x92, 0xd4, 0xde, 0xad, 0xac, 0x67, 0xe5, 0x02, 0x34, 0x46, 0x19, 0x54, 0xef, + 0x04, 0x3a, 0x33, 0x24, 0x73, 0x72, 0xd0, 0x66, 0x36, 0x07, 0xd5, 0xef, 0x19, 0x52, 0x50, 0x76, + 0x66, 0x36, 0x2f, 0xfd, 0xb6, 0x08, 0x8d, 0x4f, 0x11, 0x7b, 0x4c, 0xe8, 0xb9, 0xd4, 0xd7, 0x80, + 0x72, 0xe0, 0xf8, 0x48, 0x71, 0x14, 0xdf, 0xc6, 0x2d, 0xa8, 0xd2, 0x4b, 0x99, 0x40, 0xd4, 0x7a, + 0x56, 0xe8, 0xa5, 0x48, 0x0c, 0xc6, 0xf3, 0x00, 0xf4, 0xd2, 0x0e, 0x1d, 0xf7, 0x1c, 0x29, 0x0f, + 0x96, 0xad, 0x1a, 0xbd, 0xec, 0x4b, 0x04, 0xdf, 0x0a, 0xf4, 0xd2, 0x46, 0x94, 0x12, 0x1a, 0xa9, + 0x5c, 0x55, 0xa5, 0x97, 0xfb, 0x02, 0x56, 0x73, 0x3d, 0x4a, 0xc2, 0x10, 0x79, 0x22, 0x47, 0x8b, + 0xb9, 0x0f, 0x25, 0x82, 0x4b, 0x65, 0x5a, 0xea, 0xb2, 0x94, 0xca, 0x52, 0xa9, 0x2c, 0x95, 0x5a, + 0x91, 0x33, 0x59, 0x56, 0x2a, 0x4b, 0xa4, 0x56, 0xa5, 0x54, 0x96, 0x91, 0xca, 0x52, 0xa9, 0x35, + 0x3d, 0x57, 0x49, 0x35, 0x7f, 0x53, 0x80, 0xf5, 0xe9, 0xc2, 0x4f, 0xd5, 0xa6, 0x6f, 0x43, 0xc3, + 0x15, 0xeb, 0x95, 0xdb, 0x93, 0x9d, 0x99, 0x95, 0xb4, 0xea, 0x6e, 0x66, 0x1b, 0xdf, 0x87, 0x66, + 0x20, 0x1d, 0x9c, 0x6c, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20, 0x03, 0x99, 0x1e, + 0x18, 0x9f, 0x53, 0xcc, 0xd0, 0x09, 0xa3, 0xc8, 0xf1, 0x9f, 0x46, 0x75, 0x6f, 0x40, 0x59, 0x54, + 0x2b, 0x7c, 0x99, 0x1a, 0x96, 0xf8, 0x36, 0x5f, 0x86, 0xd5, 0x9c, 0x14, 0x65, 0xeb, 0x0a, 0x94, + 0xc6, 0x28, 0x10, 0xdc, 0x9b, 0x16, 0xff, 0x34, 0x1d, 0xe8, 0x58, 0xc8, 0xf1, 0x9e, 0x9e, 0x36, + 0x4a, 0x44, 0x29, 0x15, 0xb1, 0x09, 0x46, 0x56, 0x84, 0x52, 0x45, 0x6b, 0x5d, 0xc8, 0x68, 0xfd, + 0x08, 0x3a, 0x7b, 0x63, 0x12, 0xa1, 0x13, 0xe6, 0xe1, 0xe0, 0x69, 0xb4, 0x23, 0xbf, 0x80, 0xd5, + 0xcf, 0xd8, 0xe4, 0x73, 0xce, 0x2c, 0xc2, 0x5f, 0xa1, 0xa7, 0x64, 0x1f, 0x25, 0x8f, 0xb5, 0x7d, + 0x94, 0x3c, 0xe6, 0xcd, 0x8d, 0x4b, 0xc6, 0xb1, 0x1f, 0x88, 0x50, 0x68, 0x5a, 0x0a, 0x32, 0x77, + 0xa1, 0x21, 0x6b, 0xe8, 0x63, 0xe2, 0xc5, 0x63, 0x34, 0x37, 0x06, 0x37, 0x00, 0x42, 0x87, 0x3a, + 0x3e, 0x62, 0x88, 0xca, 0x3d, 0x54, 0xb3, 0x32, 0x18, 0xf3, 0x77, 0x45, 0x58, 0x93, 0xf7, 0x0d, + 0x27, 0xb2, 0xcd, 0xd6, 0x26, 0xf4, 0xa0, 0x3a, 0x22, 0x11, 0xcb, 0x30, 0x4c, 0x60, 0xae, 0x22, + 0xef, 0xcf, 0x25, 0x37, 0xfe, 0x99, 0xbb, 0x04, 0x28, 0x5d, 0x7d, 0x09, 0x30, 0xd3, 0xe6, 0x97, + 0x67, 0xdb, 0x7c, 0x1e, 0x6d, 0x9a, 0x08, 0xcb, 0x18, 0xaf, 0x59, 0x35, 0x85, 0x39, 0xf2, 0x8c, + 0xbb, 0xd0, 0x1e, 0x72, 0x2d, 0xed, 0x11, 0x21, 0xe7, 0x76, 0xe8, 0xb0, 0x91, 0x08, 0xf5, 0x9a, + 0xd5, 0x14, 0xe8, 0x43, 0x42, 0xce, 0xfb, 0x0e, 0x1b, 0x19, 0xef, 0x42, 0x4b, 0x95, 0x81, 0xbe, + 0x70, 0x51, 0xa4, 0x0e, 0x3f, 0x15, 0x45, 0x59, 0xef, 0x59, 0xcd, 0xf3, 0x0c, 0x14, 0x99, 0x37, + 0xe1, 0xc6, 0x43, 0x14, 0x31, 0x4a, 0x26, 0x79, 0xc7, 0x98, 0xdf, 0x07, 0x38, 0x0a, 0x18, 0xa2, + 0x67, 0x8e, 0x8b, 0x22, 0xe3, 0x8d, 0x2c, 0xa4, 0x8a, 0xa3, 0x95, 0x2d, 0x79, 0xdd, 0x93, 0x0c, + 0x58, 0x19, 0x1a, 0x73, 0x0b, 0x96, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0xa2, 0xfe, 0x52, 0xf3, 0x1a, + 0x6a, 0x9e, 0x40, 0x5a, 0x6a, 0xcc, 0x3c, 0xd4, 0x2d, 0x6c, 0xca, 0x4e, 0x2d, 0xd1, 0x16, 0xd4, + 0xb0, 0xc6, 0xa9, 0xac, 0x32, 0x2b, 0x3a, 0x25, 0x31, 0xdf, 0x87, 0x55, 0xc9, 0x49, 0x72, 0xd6, + 0x6c, 0x5e, 0x84, 0x65, 0xaa, 0xd5, 0x28, 0xa4, 0xf7, 0x3c, 0x8a, 0x48, 0x8d, 0x71, 0x7f, 0x7c, + 0x82, 0x23, 0x96, 0x1a, 0xa2, 0xfd, 0xb1, 0x0a, 0x1d, 0x3e, 0x90, 0xe3, 0x69, 0x7e, 0x08, 0x8d, + 0x1d, 0xab, 0xff, 0x29, 0xc2, 0xc3, 0xd1, 0x80, 0x67, 0xcf, 0xef, 0xe5, 0x61, 0x65, 0xb0, 0xa1, + 0xb4, 0xcd, 0x0c, 0x59, 0x39, 0x3a, 0xf3, 0x23, 0x58, 0xdf, 0xf1, 0xbc, 0x2c, 0x4a, 0x6b, 0xfd, + 0x06, 0xd4, 0x82, 0x0c, 0xbb, 0xcc, 0x99, 0x95, 0xa3, 0x4e, 0x89, 0xcc, 0x9f, 0xc1, 0xea, 0xa3, + 0x60, 0x8c, 0x03, 0xb4, 0xd7, 0x3f, 0x3d, 0x46, 0x49, 0x2e, 0x32, 0xa0, 0xcc, 0x6b, 0x36, 0xc1, + 0xa3, 0x6a, 0x89, 0x6f, 0x1e, 0x9c, 0xc1, 0xc0, 0x76, 0xc3, 0x38, 0x52, 0x97, 0x3d, 0xcb, 0xc1, + 0x60, 0x2f, 0x8c, 0x23, 0x7e, 0xb8, 0xf0, 0xe2, 0x82, 0x04, 0xe3, 0x89, 0x88, 0xd0, 0xaa, 0x55, + 0x71, 0xc3, 0xf8, 0x51, 0x30, 0x9e, 0x98, 0xdf, 0x15, 0x1d, 0x38, 0x42, 0x9e, 0xe5, 0x04, 0x1e, + 0xf1, 0x1f, 0xa2, 0x8b, 0x8c, 0x84, 0xa4, 0xdb, 0xd3, 0x99, 0xe8, 0xeb, 0x02, 0x34, 0x76, 0x86, + 0x28, 0x60, 0x0f, 0x11, 0x73, 0xf0, 0x58, 0x74, 0x74, 0x17, 0x88, 0x46, 0x98, 0x04, 0x2a, 0xdc, + 0x34, 0xc8, 0x1b, 0x72, 0x1c, 0x60, 0x66, 0x7b, 0x0e, 0xf2, 0x49, 0x20, 0xb8, 0x54, 0x2d, 0xe0, + 0xa8, 0x87, 0x02, 0x63, 0xbc, 0x0c, 0x6d, 0x79, 0x19, 0x67, 0x8f, 0x9c, 0xc0, 0x1b, 0xf3, 0x40, + 0x2f, 0x89, 0xd0, 0x6c, 0x49, 0xf4, 0xa1, 0xc2, 0x1a, 0xaf, 0xc0, 0x8a, 0x0a, 0xc3, 0x94, 0xb2, + 0x2c, 0x28, 0xdb, 0x0a, 0x9f, 0x23, 0x8d, 0xc3, 0x90, 0x50, 0x16, 0xd9, 0x11, 0x72, 0x5d, 0xe2, + 0x87, 0xaa, 0x1d, 0x6a, 0x6b, 0xfc, 0x89, 0x44, 0x9b, 0x43, 0x58, 0x3d, 0xe0, 0x76, 0x2a, 0x4b, + 0xd2, 0x6d, 0xd5, 0xf2, 0x91, 0x6f, 0x0f, 0xc6, 0xc4, 0x3d, 0xb7, 0x79, 0x72, 0x54, 0x1e, 0xe6, + 0x05, 0xd7, 0x2e, 0x47, 0x9e, 0xe0, 0xaf, 0x44, 0xe7, 0xcf, 0xa9, 0x46, 0x84, 0x85, 0xe3, 0x78, + 0x68, 0x87, 0x94, 0x0c, 0x90, 0x32, 0xb1, 0xed, 0x23, 0xff, 0x50, 0xe2, 0xfb, 0x1c, 0x6d, 0xfe, + 0xa9, 0x00, 0x6b, 0x79, 0x49, 0x2a, 0xd5, 0x6f, 0xc3, 0x5a, 0x5e, 0x94, 0x3a, 0xfe, 0x65, 0x79, + 0xd9, 0xc9, 0x0a, 0x94, 0x85, 0xc0, 0x7d, 0x68, 0x8a, 0xab, 0x5b, 0xdb, 0x93, 0x9c, 0xf2, 0x45, + 0x4f, 0x76, 0x5d, 0xac, 0x86, 0x93, 0x5d, 0xa5, 0x77, 0xe1, 0x96, 0x32, 0xdf, 0x9e, 0x55, 0x5b, + 0x6e, 0x88, 0x75, 0x45, 0x70, 0x3c, 0xa5, 0xfd, 0x27, 0xd0, 0x4d, 0x51, 0xbb, 0x13, 0x81, 0x4c, + 0x37, 0xf3, 0xea, 0x94, 0xb1, 0x3b, 0x9e, 0x47, 0x45, 0x94, 0x94, 0xad, 0x79, 0x43, 0xe6, 0x03, + 0xb8, 0x79, 0x82, 0x98, 0xf4, 0x86, 0xc3, 0x54, 0x27, 0x22, 0x99, 0xad, 0x40, 0xe9, 0x04, 0xb9, + 0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0xf9, 0x06, 0x3c, 0x8d, 0x90, 0x2b, 0xac, 0x2c, 0x59, 0xe2, 0xdb, + 0x0c, 0xa1, 0xf2, 0xe1, 0xc9, 0x01, 0xaf, 0x37, 0xf8, 0xa6, 0x96, 0xf5, 0x89, 0x3a, 0x8b, 0x9a, + 0x56, 0x45, 0xc0, 0x47, 0x9e, 0xf1, 0x11, 0xac, 0xca, 0x21, 0x77, 0xe4, 0x04, 0x43, 0x64, 0x87, + 0x64, 0x8c, 0x5d, 0xb9, 0xf5, 0x5b, 0xf7, 0x7a, 0x2a, 0x7c, 0x15, 0x9f, 0x3d, 0x41, 0xd2, 0x17, + 0x14, 0x56, 0x67, 0x38, 0x8d, 0x32, 0xff, 0x51, 0x80, 0x8a, 0x3a, 0x0e, 0xf8, 0x91, 0xe6, 0x51, + 0x7c, 0x81, 0xa8, 0xda, 0xec, 0x0a, 0x32, 0x5e, 0x82, 0x96, 0xfc, 0xb2, 0x49, 0xc8, 0x30, 0x49, + 0x0e, 0x99, 0xa6, 0xc4, 0x3e, 0x92, 0x48, 0x71, 0xdd, 0x27, 0x2e, 0xdc, 0x54, 0x6f, 0xab, 0x20, + 0x8e, 0x3f, 0x8b, 0xb8, 0x52, 0xe2, 0x50, 0xa9, 0x59, 0x0a, 0xe2, 0xc1, 0xa5, 0xf9, 0x2d, 0x09, + 0x7e, 0x1a, 0xe4, 0xc1, 0xe5, 0x93, 0x38, 0x60, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0x29, 0x02, 0x02, + 0xd5, 0xe7, 0x18, 0x63, 0x13, 0xaa, 0x67, 0x91, 0x2d, 0xac, 0x11, 0x15, 0x63, 0x72, 0xb2, 0x29, + 0xab, 0xad, 0xca, 0x59, 0x24, 0x3e, 0xcc, 0x5f, 0x17, 0x60, 0x59, 0x5e, 0x8e, 0xf3, 0xbe, 0x3b, + 0x39, 0xf5, 0x8b, 0x58, 0x54, 0x50, 0x42, 0x2b, 0x79, 0xd2, 0x8b, 0x6f, 0x9e, 0x63, 0x2e, 0x7c, + 0x79, 0x76, 0x29, 0x23, 0x2e, 0x7c, 0x71, 0x68, 0xbd, 0x04, 0xad, 0xb4, 0x78, 0x10, 0xe3, 0xd2, + 0x98, 0x66, 0x82, 0x15, 0x64, 0x0b, 0x6d, 0x32, 0x7f, 0x0c, 0x90, 0x5e, 0x12, 0xf3, 0xed, 0x10, + 0x27, 0xca, 0xf0, 0x4f, 0x8e, 0x19, 0x26, 0x65, 0x07, 0xff, 0x34, 0xee, 0x42, 0xcb, 0xf1, 0x3c, + 0xcc, 0xa7, 0x3b, 0xe3, 0x03, 0xec, 0x25, 0x09, 0x24, 0x8f, 0x35, 0xff, 0x5a, 0x80, 0xf6, 0x1e, + 0x09, 0x27, 0x1f, 0xe2, 0x31, 0xca, 0x64, 0x37, 0xa1, 0xa4, 0xaa, 0x3a, 0xf8, 0x37, 0xaf, 0xa4, + 0xcf, 0xf0, 0x18, 0xc9, 0xb0, 0x97, 0xbb, 0xae, 0xca, 0x11, 0x22, 0xe4, 0xf5, 0x60, 0x72, 0x25, + 0xd8, 0x94, 0x83, 0xc7, 0xc4, 0x13, 0x3d, 0x83, 0x87, 0xa9, 0x9d, 0x5c, 0x00, 0x36, 0xad, 0x8a, + 0x87, 0xa9, 0x18, 0x52, 0x86, 0x2c, 0x89, 0x0b, 0xde, 0xac, 0x21, 0xcb, 0x12, 0xc3, 0x0d, 0x59, + 0x87, 0x65, 0x72, 0x76, 0x16, 0x21, 0x26, 0xd6, 0xaa, 0x64, 0x29, 0x28, 0x49, 0xc1, 0xd5, 0x4c, + 0x0a, 0x5e, 0x03, 0xe3, 0x00, 0xb1, 0x47, 0x8f, 0x8e, 0xf7, 0x2f, 0x50, 0xc0, 0xf4, 0xc9, 0xf5, + 0x3a, 0x54, 0x35, 0xea, 0xbf, 0xb9, 0x3a, 0x7d, 0x15, 0x5a, 0x3b, 0x9e, 0x77, 0xf2, 0xd8, 0x09, + 0xb5, 0x3f, 0xba, 0x50, 0xe9, 0xef, 0x1d, 0xf5, 0xa5, 0x4b, 0x4a, 0xdc, 0x00, 0x05, 0xf2, 0x93, + 0xf2, 0x00, 0xb1, 0x63, 0xc4, 0x28, 0x76, 0x93, 0x93, 0xf2, 0x0e, 0x54, 0x14, 0x86, 0xcf, 0xf4, + 0xe5, 0xa7, 0x3e, 0x02, 0x14, 0x68, 0xfe, 0x00, 0x8c, 0x1f, 0xf1, 0x9a, 0x0f, 0xc9, 0x82, 0x5f, + 0x49, 0x7a, 0x15, 0x3a, 0x17, 0x02, 0x6b, 0xcb, 0x62, 0x28, 0xb3, 0x0c, 0x6d, 0x39, 0x20, 0xf2, + 0x83, 0x90, 0x7d, 0x0a, 0xab, 0xb2, 0x44, 0x95, 0x7c, 0xae, 0xc1, 0x82, 0xfb, 0x30, 0x59, 0xcf, + 0xb2, 0x25, 0xbe, 0xef, 0xfd, 0xb9, 0xa3, 0x8e, 0x31, 0x75, 0x23, 0x62, 0x1c, 0x40, 0x7b, 0xea, + 0xf9, 0xca, 0x50, 0x57, 0x64, 0xf3, 0x5f, 0xb5, 0x7a, 0xeb, 0x5b, 0xf2, 0x39, 0x6c, 0x4b, 0x3f, + 0x87, 0x6d, 0xed, 0xfb, 0x21, 0x9b, 0x18, 0xfb, 0xd0, 0xca, 0x3f, 0xf4, 0x18, 0xcf, 0xea, 0x8a, + 0x72, 0xce, 0xf3, 0xcf, 0x42, 0x36, 0x07, 0xd0, 0x9e, 0x7a, 0xf3, 0xd1, 0xfa, 0xcc, 0x7f, 0x0a, + 0x5a, 0xc8, 0xe8, 0x01, 0xd4, 0x33, 0x8f, 0x3c, 0x46, 0x57, 0x32, 0x99, 0x7d, 0xf7, 0x59, 0xc8, + 0x60, 0x0f, 0x9a, 0xb9, 0x77, 0x17, 0xa3, 0xa7, 0xec, 0x99, 0xf3, 0x18, 0xb3, 0x90, 0xc9, 0x2e, + 0xd4, 0x33, 0xcf, 0x1f, 0x5a, 0x8b, 0xd9, 0x37, 0x96, 0xde, 0xad, 0x39, 0x23, 0xea, 0xb4, 0x3c, + 0x80, 0xf6, 0xd4, 0x9b, 0x88, 0x76, 0xc9, 0xfc, 0xa7, 0x92, 0x85, 0xca, 0x7c, 0x2c, 0x96, 0x28, + 0xd3, 0xf2, 0x66, 0x96, 0x68, 0xf6, 0x05, 0xa4, 0xf7, 0xdc, 0xfc, 0x41, 0xa5, 0xd5, 0x3e, 0xb4, + 0xf2, 0x8f, 0x1f, 0x9a, 0xd9, 0xdc, 0x27, 0x91, 0xab, 0xd7, 0x3b, 0xf7, 0x0e, 0x92, 0xae, 0xf7, + 0xbc, 0xe7, 0x91, 0x85, 0x8c, 0x76, 0x00, 0x54, 0x83, 0xeb, 0xe1, 0x20, 0x71, 0xf4, 0x4c, 0x63, + 0x9d, 0x38, 0x7a, 0x4e, 0x33, 0xfc, 0x00, 0x40, 0xf6, 0xa5, 0x1e, 0x89, 0x99, 0x71, 0x53, 0xab, + 0x31, 0xd5, 0x0c, 0xf7, 0xba, 0xb3, 0x03, 0x33, 0x0c, 0x10, 0xa5, 0xd7, 0x61, 0xf0, 0x01, 0x40, + 0xda, 0xef, 0x6a, 0x06, 0x33, 0x1d, 0xf0, 0x15, 0x3e, 0x68, 0x64, 0xbb, 0x5b, 0x43, 0xd9, 0x3a, + 0xa7, 0xe3, 0xbd, 0x82, 0x45, 0x7b, 0xaa, 0x7b, 0xc9, 0x6f, 0xb6, 0xe9, 0xa6, 0xa6, 0x37, 0xd3, + 0xc1, 0x18, 0xf7, 0xa1, 0x91, 0x6d, 0x5b, 0xb4, 0x16, 0x73, 0x5a, 0x99, 0x5e, 0xae, 0x75, 0x31, + 0x1e, 0x40, 0x2b, 0xdf, 0xb2, 0xe8, 0x2d, 0x35, 0xb7, 0x91, 0xe9, 0xa9, 0x0b, 0xb9, 0x0c, 0xf9, + 0x5b, 0x00, 0x69, 0x6b, 0xa3, 0xdd, 0x37, 0xd3, 0xec, 0x4c, 0x49, 0x3d, 0x80, 0xf6, 0x54, 0xcb, + 0xa2, 0x2d, 0x9e, 0xdf, 0xc9, 0x2c, 0x74, 0xdd, 0xdb, 0x00, 0xe9, 0x71, 0xa1, 0xa5, 0xcf, 0x1c, + 0x20, 0xbd, 0xa6, 0xbe, 0xac, 0x94, 0x74, 0x7b, 0xd0, 0xcc, 0xf5, 0xf3, 0x3a, 0xcd, 0xcc, 0x6b, + 0xf2, 0xaf, 0x4a, 0xbe, 0xf9, 0xe6, 0x57, 0x7b, 0x6e, 0x6e, 0x4b, 0x7c, 0xd5, 0xfe, 0xc9, 0x76, + 0x5c, 0x7a, 0xe5, 0xe6, 0x74, 0x61, 0xdf, 0x12, 0xcf, 0xd9, 0xae, 0x2a, 0x13, 0xcf, 0x73, 0x9a, + 0xad, 0x85, 0x8c, 0x0e, 0xa1, 0x7d, 0xa0, 0x0b, 0x66, 0x55, 0xcc, 0x2b, 0x75, 0xe6, 0x34, 0x2f, + 0xbd, 0xde, 0xbc, 0x21, 0x15, 0x54, 0x1f, 0x43, 0x67, 0xa6, 0x90, 0x37, 0x36, 0x92, 0x2b, 0xe3, + 0xb9, 0x15, 0xfe, 0x42, 0xb5, 0x8e, 0x60, 0x65, 0xba, 0x8e, 0x37, 0x9e, 0x57, 0x89, 0x72, 0x7e, + 0x7d, 0xbf, 0x90, 0xd5, 0xbb, 0x50, 0xd5, 0xb5, 0x99, 0xa1, 0xae, 0xe6, 0xa7, 0x6a, 0xb5, 0x85, + 0x53, 0xef, 0x43, 0x3d, 0x53, 0x0a, 0xe9, 0x6c, 0x37, 0x5b, 0x1d, 0xf5, 0xd4, 0x4d, 0x7a, 0x42, + 0x79, 0x1f, 0x2a, 0xaa, 0xfc, 0x31, 0xd6, 0x92, 0x4d, 0x9e, 0xa9, 0x86, 0xae, 0xda, 0x61, 0x07, + 0x88, 0x65, 0x8a, 0x1a, 0x2d, 0x74, 0xb6, 0xce, 0xd1, 0x29, 0x36, 0x37, 0xa2, 0xd6, 0x62, 0x07, + 0x1a, 0xd9, 0xb2, 0x46, 0x2f, 0xe9, 0x9c, 0x52, 0x67, 0x91, 0x26, 0xbb, 0x97, 0x5f, 0x7f, 0xb3, + 0xf1, 0xcc, 0xdf, 0xbf, 0xd9, 0x78, 0xe6, 0x57, 0x4f, 0x36, 0x0a, 0x5f, 0x3f, 0xd9, 0x28, 0xfc, + 0xed, 0xc9, 0x46, 0xe1, 0x5f, 0x4f, 0x36, 0x0a, 0x3f, 0xf9, 0xf9, 0xff, 0xf8, 0x1f, 0x21, 0x1a, + 0x07, 0x0c, 0xfb, 0x68, 0xfb, 0x02, 0x53, 0x96, 0x19, 0x0a, 0xcf, 0x87, 0xf2, 0x8f, 0x42, 0x99, + 0xff, 0x0f, 0x71, 0x2d, 0x07, 0xcb, 0x02, 0x7e, 0xeb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x6c, + 0xf4, 0x1d, 0x49, 0x8c, 0x24, 0x00, 0x00, } func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { @@ -5108,6 +5163,43 @@ func (m *SetGuestDateTimeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) return len(dAtA) - i, nil } +func (m *FSGroup) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FSGroup) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FSGroup) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.GroupChangePolicy != 0 { + i = encodeVarintAgent(dAtA, i, uint64(m.GroupChangePolicy)) + i-- + dAtA[i] = 0x18 + } + if m.GroupId != 0 { + i = encodeVarintAgent(dAtA, i, uint64(m.GroupId)) + i-- + dAtA[i] = 0x10 + } + return len(dAtA) - i, nil +} + func (m *Storage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -5132,6 +5224,18 @@ func (m *Storage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FsGroup != nil { + { + size, err := m.FsGroup.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintAgent(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } if len(m.MountPoint) > 0 { i -= len(m.MountPoint) copy(dAtA[i:], m.MountPoint) @@ -5452,20 +5556,20 @@ func (m *AddSwapRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.PCIPath) > 0 { - dAtA26 := make([]byte, len(m.PCIPath)*10) - var j25 int + dAtA27 := make([]byte, len(m.PCIPath)*10) + var j26 int for _, num := range m.PCIPath { for num >= 1<<7 { - dAtA26[j25] = uint8(uint64(num)&0x7f | 0x80) + dAtA27[j26] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j25++ + j26++ } - dAtA26[j25] = uint8(num) - j25++ + dAtA27[j26] = uint8(num) + j26++ } - i -= j25 - copy(dAtA[i:], dAtA26[:j25]) - i = encodeVarintAgent(dAtA, i, uint64(j25)) + i -= j26 + copy(dAtA[i:], dAtA27[:j26]) + i = encodeVarintAgent(dAtA, i, uint64(j26)) i-- dAtA[i] = 0xa } @@ -6684,6 +6788,24 @@ func (m *SetGuestDateTimeRequest) Size() (n int) { return n } +func (m *FSGroup) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.GroupId != 0 { + n += 1 + sovAgent(uint64(m.GroupId)) + } + if m.GroupChangePolicy != 0 { + n += 1 + sovAgent(uint64(m.GroupChangePolicy)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *Storage) Size() (n int) { if m == nil { return 0 @@ -6718,6 +6840,10 @@ func (m *Storage) Size() (n int) { if l > 0 { n += 1 + l + sovAgent(uint64(l)) } + if m.FsGroup != nil { + l = m.FsGroup.Size() + n += 1 + l + sovAgent(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7631,6 +7757,18 @@ func (this *SetGuestDateTimeRequest) String() string { }, "") return s } +func (this *FSGroup) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&FSGroup{`, + `GroupId:` + fmt.Sprintf("%v", this.GroupId) + `,`, + `GroupChangePolicy:` + fmt.Sprintf("%v", this.GroupChangePolicy) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} func (this *Storage) String() string { if this == nil { return "nil" @@ -7642,6 +7780,7 @@ func (this *Storage) String() string { `Fstype:` + fmt.Sprintf("%v", this.Fstype) + `,`, `Options:` + fmt.Sprintf("%v", this.Options) + `,`, `MountPoint:` + fmt.Sprintf("%v", this.MountPoint) + `,`, + `FsGroup:` + strings.Replace(this.FsGroup.String(), "FSGroup", "FSGroup", 1) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -14422,6 +14561,95 @@ func (m *SetGuestDateTimeRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *FSGroup) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FSGroup: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FSGroup: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupId", wireType) + } + m.GroupId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.GroupId |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupChangePolicy", wireType) + } + m.GroupChangePolicy = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.GroupChangePolicy |= protocols.FSGroupChangePolicy(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Storage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -14643,6 +14871,42 @@ func (m *Storage) Unmarshal(dAtA []byte) error { } m.MountPoint = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FsGroup", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthAgent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FsGroup == nil { + m.FsGroup = &FSGroup{} + } + if err := m.FsGroup.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipAgent(dAtA[iNdEx:]) diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go index f4dd26cfb..8646f4bd1 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go @@ -49,6 +49,35 @@ func (IPFamily) EnumDescriptor() ([]byte, []int) { return fileDescriptor_f715d0876e8f65d3, []int{0} } +// FSGroupChangePolicy defines the policy for applying group id ownership change on a mounted volume. +type FSGroupChangePolicy int32 + +const ( + // Always indicates that the volume ownership will always be changed. + FSGroupChangePolicy_Always FSGroupChangePolicy = 0 + // OnRootMismatch indicates that the volume ownership will be changed only + // when the ownership of the root directory does not match with the expected group id for the volume. + FSGroupChangePolicy_OnRootMismatch FSGroupChangePolicy = 1 +) + +var FSGroupChangePolicy_name = map[int32]string{ + 0: "Always", + 1: "OnRootMismatch", +} + +var FSGroupChangePolicy_value = map[string]int32{ + "Always": 0, + "OnRootMismatch": 1, +} + +func (x FSGroupChangePolicy) String() string { + return proto.EnumName(FSGroupChangePolicy_name, int32(x)) +} + +func (FSGroupChangePolicy) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_f715d0876e8f65d3, []int{1} +} + type IPAddress struct { Family IPFamily `protobuf:"varint,1,opt,name=family,proto3,enum=types.IPFamily" json:"family,omitempty"` Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` @@ -230,6 +259,7 @@ var xxx_messageInfo_ARPNeighbor proto.InternalMessageInfo func init() { proto.RegisterEnum("types.IPFamily", IPFamily_name, IPFamily_value) + proto.RegisterEnum("types.FSGroupChangePolicy", FSGroupChangePolicy_name, FSGroupChangePolicy_value) proto.RegisterType((*IPAddress)(nil), "types.IPAddress") proto.RegisterType((*Interface)(nil), "types.Interface") proto.RegisterType((*Route)(nil), "types.Route") @@ -241,38 +271,40 @@ func init() { } var fileDescriptor_f715d0876e8f65d3 = []byte{ - // 482 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0x3f, 0x8b, 0xdb, 0x30, - 0x18, 0xc6, 0xa3, 0x38, 0xf6, 0xc5, 0x0a, 0xd7, 0x06, 0x51, 0x0e, 0xd1, 0x82, 0x31, 0x59, 0x6a, - 0x0a, 0x8d, 0x21, 0x2d, 0xdd, 0xaf, 0xc3, 0x41, 0x96, 0x62, 0xb4, 0xb5, 0x4b, 0x91, 0x1d, 0xc5, - 0x31, 0xb1, 0x2d, 0x23, 0xc9, 0x09, 0xd9, 0xfa, 0x45, 0xba, 0xf5, 0xc3, 0xdc, 0xd8, 0xb1, 0xe3, - 0x25, 0x9f, 0xa4, 0x48, 0x72, 0x52, 0xf7, 0x0f, 0x85, 0x9b, 0xf2, 0xfe, 0x5e, 0x49, 0x79, 0x9f, - 0xe7, 0x91, 0x05, 0x93, 0xbc, 0x50, 0x9b, 0x36, 0x9d, 0x67, 0xbc, 0x8a, 0xb7, 0x54, 0xd1, 0xd7, - 0x19, 0xaf, 0x15, 0x2d, 0x6a, 0x26, 0xe4, 0x5f, 0x2c, 0x45, 0x16, 0x97, 0x45, 0x2a, 0xe3, 0x46, - 0x70, 0xc5, 0x33, 0x5e, 0x76, 0x95, 0x8c, 0xd5, 0xa1, 0x61, 0x72, 0x6e, 0x00, 0xb9, 0x06, 0x66, - 0x29, 0xf4, 0x97, 0xc9, 0xed, 0x6a, 0x25, 0x98, 0x94, 0xe8, 0x25, 0xf4, 0xd6, 0xb4, 0x2a, 0xca, - 0x03, 0x06, 0x21, 0x88, 0x9e, 0x2c, 0x9e, 0xce, 0xed, 0x89, 0x65, 0x72, 0x67, 0xda, 0xa4, 0x5b, - 0x46, 0x18, 0x5e, 0x51, 0x7b, 0x06, 0x0f, 0x43, 0x10, 0xf9, 0xe4, 0x8c, 0x08, 0xc1, 0x51, 0x45, - 0xe5, 0x16, 0x3b, 0xa6, 0x6d, 0xea, 0xd9, 0x03, 0x80, 0xfe, 0xb2, 0x56, 0x4c, 0xac, 0x69, 0xc6, - 0xd0, 0x0d, 0xf4, 0x56, 0x6c, 0x57, 0x64, 0xcc, 0x0c, 0xf1, 0x49, 0x47, 0xfa, 0x64, 0x4d, 0x2b, - 0xd6, 0xfd, 0xa1, 0xa9, 0xd1, 0x02, 0x4e, 0x2e, 0xea, 0x98, 0xc4, 0x4e, 0xe8, 0x44, 0x93, 0xc5, - 0xf4, 0xa2, 0xaa, 0x5b, 0x21, 0xfd, 0x4d, 0x68, 0x0a, 0x9d, 0x4a, 0xb5, 0x78, 0x14, 0x82, 0x68, - 0x44, 0x74, 0xa9, 0x27, 0x6e, 0xf6, 0x7a, 0x03, 0x76, 0xed, 0x44, 0x4b, 0xda, 0x45, 0x93, 0x15, - 0x09, 0x55, 0x1b, 0xec, 0x59, 0x17, 0x1d, 0x6a, 0x2d, 0x7a, 0x06, 0xbe, 0xb2, 0x5a, 0x74, 0x8d, - 0x5e, 0x40, 0x5f, 0xd0, 0xfd, 0xe7, 0x75, 0x49, 0x73, 0x89, 0xc7, 0x21, 0x88, 0xae, 0xc9, 0x58, - 0xd0, 0xfd, 0x9d, 0xe6, 0xd9, 0x37, 0x00, 0x5d, 0xc2, 0x5b, 0x65, 0x6c, 0xac, 0x98, 0x54, 0x9d, - 0x39, 0x53, 0xeb, 0x41, 0x39, 0x55, 0x6c, 0x4f, 0x0f, 0xe7, 0xb8, 0x3a, 0xec, 0x85, 0xe1, 0xfc, - 0x16, 0xc6, 0x0d, 0xf4, 0x24, 0x6f, 0x45, 0xc6, 0x8c, 0x0f, 0x9f, 0x74, 0x84, 0x9e, 0x41, 0x57, - 0x66, 0xbc, 0x61, 0xc6, 0xc9, 0x35, 0xb1, 0xd0, 0xbb, 0x37, 0xef, 0xbf, 0xf7, 0x36, 0xfb, 0x0a, - 0xe0, 0xe4, 0x96, 0x24, 0x1f, 0x58, 0x91, 0x6f, 0x52, 0x2e, 0x74, 0xbe, 0x8a, 0x5f, 0xc2, 0x33, - 0x9a, 0xff, 0x99, 0x6f, 0x6f, 0x53, 0x4f, 0xf2, 0xf0, 0x4f, 0xc9, 0x65, 0xa9, 0x3f, 0x83, 0xb3, - 0x15, 0x4b, 0x46, 0xb2, 0xa2, 0xca, 0x3a, 0x71, 0x89, 0x05, 0xdd, 0xb5, 0x49, 0xba, 0xb6, 0x6b, - 0xe0, 0xd5, 0x73, 0x38, 0x3e, 0x6b, 0x46, 0x1e, 0x1c, 0xee, 0xde, 0x4e, 0x07, 0xe6, 0xf7, 0xdd, - 0x14, 0xbc, 0x97, 0xf7, 0xc7, 0x60, 0xf0, 0xe3, 0x18, 0x0c, 0xbe, 0x9c, 0x02, 0x70, 0x7f, 0x0a, - 0xc0, 0xf7, 0x53, 0x00, 0x1e, 0x4e, 0x01, 0xf8, 0xf4, 0xf1, 0x91, 0x8f, 0x43, 0xb4, 0xb5, 0x2a, - 0x2a, 0x16, 0xef, 0x0a, 0xa1, 0x7a, 0x4b, 0xcd, 0x36, 0x8f, 0x69, 0xce, 0x6a, 0xf5, 0xeb, 0xe1, - 0xa4, 0x9e, 0x29, 0xdf, 0xfc, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xae, 0x92, 0x74, 0xed, 0x80, 0x03, - 0x00, 0x00, + // 527 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xcd, 0x8e, 0xd3, 0x30, + 0x14, 0x85, 0xeb, 0x69, 0x93, 0x69, 0x5c, 0xcd, 0x10, 0x19, 0x34, 0x8a, 0x40, 0x8a, 0xaa, 0x6e, + 0xa8, 0x46, 0xa2, 0x91, 0xca, 0xcf, 0xbe, 0x20, 0x15, 0x75, 0x01, 0x44, 0x66, 0x05, 0x1b, 0xe4, + 0xa6, 0x6e, 0x62, 0x35, 0x89, 0x23, 0xdb, 0x69, 0xd5, 0x1d, 0x2f, 0xc2, 0x8e, 0x87, 0x99, 0x25, + 0x4b, 0x96, 0x33, 0x7d, 0x12, 0x64, 0x3b, 0x2d, 0xe1, 0x47, 0x48, 0xac, 0x7a, 0xbf, 0x6b, 0xbb, + 0xf7, 0x9c, 0xe3, 0x18, 0xc6, 0x29, 0x53, 0x59, 0xbd, 0x9c, 0x24, 0xbc, 0x88, 0x36, 0x44, 0x91, + 0x27, 0x09, 0x2f, 0x15, 0x61, 0x25, 0x15, 0xf2, 0x0f, 0x96, 0x22, 0x89, 0x72, 0xb6, 0x94, 0x51, + 0x25, 0xb8, 0xe2, 0x09, 0xcf, 0x9b, 0x4a, 0x46, 0x6a, 0x5f, 0x51, 0x39, 0x31, 0x80, 0x1c, 0x03, + 0xa3, 0x25, 0xf4, 0x16, 0xf1, 0x6c, 0xb5, 0x12, 0x54, 0x4a, 0xf4, 0x18, 0xba, 0x6b, 0x52, 0xb0, + 0x7c, 0x1f, 0x80, 0x21, 0x18, 0x5f, 0x4e, 0xef, 0x4d, 0xec, 0x89, 0x45, 0x3c, 0x37, 0x6d, 0xdc, + 0x2c, 0xa3, 0x00, 0x9e, 0x13, 0x7b, 0x26, 0x38, 0x1b, 0x82, 0xb1, 0x87, 0x8f, 0x88, 0x10, 0xec, + 0x15, 0x44, 0x6e, 0x82, 0xae, 0x69, 0x9b, 0x7a, 0x74, 0x0b, 0xa0, 0xb7, 0x28, 0x15, 0x15, 0x6b, + 0x92, 0x50, 0x74, 0x05, 0xdd, 0x15, 0xdd, 0xb2, 0x84, 0x9a, 0x21, 0x1e, 0x6e, 0x48, 0x9f, 0x2c, + 0x49, 0x41, 0x9b, 0x3f, 0x34, 0x35, 0x9a, 0xc2, 0xc1, 0x49, 0x1d, 0x95, 0x41, 0x77, 0xd8, 0x1d, + 0x0f, 0xa6, 0xfe, 0x49, 0x55, 0xb3, 0x82, 0xdb, 0x9b, 0x90, 0x0f, 0xbb, 0x85, 0xaa, 0x83, 0xde, + 0x10, 0x8c, 0x7b, 0x58, 0x97, 0x7a, 0x62, 0xb6, 0xd3, 0x1b, 0x02, 0xc7, 0x4e, 0xb4, 0xa4, 0x5d, + 0x54, 0x09, 0x8b, 0x89, 0xca, 0x02, 0xd7, 0xba, 0x68, 0x50, 0x6b, 0xd1, 0x33, 0x82, 0x73, 0xab, + 0x45, 0xd7, 0xe8, 0x11, 0xf4, 0x04, 0xd9, 0x7d, 0x5a, 0xe7, 0x24, 0x95, 0x41, 0x7f, 0x08, 0xc6, + 0x17, 0xb8, 0x2f, 0xc8, 0x6e, 0xae, 0x79, 0xf4, 0x15, 0x40, 0x07, 0xf3, 0x5a, 0x19, 0x1b, 0x2b, + 0x2a, 0x55, 0x63, 0xce, 0xd4, 0x7a, 0x50, 0x4a, 0x14, 0xdd, 0x91, 0xfd, 0x31, 0xae, 0x06, 0x5b, + 0x61, 0x74, 0x7f, 0x09, 0xe3, 0x0a, 0xba, 0x92, 0xd7, 0x22, 0xa1, 0xc6, 0x87, 0x87, 0x1b, 0x42, + 0x0f, 0xa0, 0x23, 0x13, 0x5e, 0x51, 0xe3, 0xe4, 0x02, 0x5b, 0x68, 0xdd, 0x9b, 0xfb, 0xcf, 0x7b, + 0x1b, 0x7d, 0x01, 0x70, 0x30, 0xc3, 0xf1, 0x5b, 0xca, 0xd2, 0x6c, 0xc9, 0x85, 0xce, 0x57, 0xf1, + 0x53, 0x78, 0x46, 0xf3, 0x5f, 0xf3, 0x6d, 0x6d, 0x6a, 0x49, 0x3e, 0xfb, 0x5d, 0x72, 0x9e, 0xeb, + 0xcf, 0xe0, 0x68, 0xc5, 0x92, 0x91, 0xac, 0x88, 0xb2, 0x4e, 0x1c, 0x6c, 0x41, 0x77, 0x6d, 0x92, + 0x8e, 0xed, 0x1a, 0xb8, 0x7e, 0x08, 0xfb, 0x47, 0xcd, 0xc8, 0x85, 0x67, 0xdb, 0x67, 0x7e, 0xc7, + 0xfc, 0xbe, 0xf0, 0xc1, 0xf5, 0x73, 0x78, 0x7f, 0xfe, 0xfe, 0xb5, 0xe0, 0x75, 0xf5, 0x2a, 0x23, + 0x65, 0x4a, 0x63, 0x9e, 0xb3, 0x64, 0x8f, 0x20, 0x74, 0x67, 0xf9, 0x8e, 0xec, 0xa5, 0xdf, 0x41, + 0x08, 0x5e, 0xbe, 0x2b, 0x31, 0xe7, 0xea, 0x0d, 0x93, 0x05, 0x51, 0x49, 0xe6, 0x83, 0x97, 0xf2, + 0xe6, 0x2e, 0xec, 0x7c, 0xbf, 0x0b, 0x3b, 0x9f, 0x0f, 0x21, 0xb8, 0x39, 0x84, 0xe0, 0xdb, 0x21, + 0x04, 0xb7, 0x87, 0x10, 0x7c, 0xfc, 0xf0, 0x9f, 0x6f, 0x4a, 0xd4, 0xa5, 0x62, 0x05, 0x8d, 0xb6, + 0x4c, 0xa8, 0xd6, 0x52, 0xb5, 0x49, 0x23, 0x92, 0xd2, 0x52, 0xfd, 0x7c, 0x6f, 0x4b, 0xd7, 0x94, + 0x4f, 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0xd7, 0x28, 0x03, 0xf1, 0xb7, 0x03, 0x00, 0x00, } func (m *IPAddress) Marshal() (dAtA []byte, err error) { From 532d53977ec276ecbb298a9755439bd3d95dbebf Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Wed, 6 Apr 2022 19:31:49 -0700 Subject: [PATCH 03/15] runtime: fsGroup support for direct-assigned volume The fsGroup will be specified by the fsGroup key in the direct-assign mountinfo metadate field. This will be set when invoking the kata-runtime binary and providing the key, value pair in the metadata field. Similarly, the fsGroupChangePolicy will also be provided in the mountinfo metadate field. Adding an extra fields FsGroup and FSGroupChangePolicy in the Mount construct for container mount which will be populated when creating block devices by parsing out the mountInfo.json. And in handleDeviceBlockVolume of the kata-agent client, it checks if the mount FSGroup is not nil, which indicates that fsGroup change is required in the guest, and will provide the FSGroup field in the protobuf to pass the value to the agent. Fixes #4018 Signed-off-by: Yibo Zhuang --- src/runtime/pkg/direct-volume/utils.go | 19 +++++++++++++++ src/runtime/pkg/direct-volume/utils_test.go | 7 +++++- src/runtime/virtcontainers/container.go | 20 ++++++++++++++++ src/runtime/virtcontainers/kata_agent.go | 16 +++++++++++++ src/runtime/virtcontainers/kata_agent_test.go | 23 +++++++++++++++++++ src/runtime/virtcontainers/mount.go | 10 ++++++++ 6 files changed, 94 insertions(+), 1 deletion(-) diff --git a/src/runtime/pkg/direct-volume/utils.go b/src/runtime/pkg/direct-volume/utils.go index 275b0508f..847b704bd 100644 --- a/src/runtime/pkg/direct-volume/utils.go +++ b/src/runtime/pkg/direct-volume/utils.go @@ -17,6 +17,25 @@ import ( const ( mountInfoFileName = "mountInfo.json" + + FSGroupMetadataKey = "fsGroup" + FSGroupChangePolicyMetadataKey = "fsGroupChangePolicy" +) + +// FSGroupChangePolicy holds policies that will be used for applying fsGroup to a volume. +// This type and the allowed values are tracking the PodFSGroupChangePolicy defined in +// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/api/core/v1/types.go +// It is up to the client using the direct-assigned volume feature (e.g. CSI drivers) to determine +// the optimal setting for this change policy (i.e. from Pod spec or assuming volume ownership +// based on the storage offering). +type FSGroupChangePolicy string + +const ( + // FSGroupChangeAlways indicates that volume's ownership should always be changed. + FSGroupChangeAlways FSGroupChangePolicy = "Always" + // FSGroupChangeOnRootMismatch indicates that volume's ownership will be changed + // only when ownership of root directory does not match with the desired group id. + FSGroupChangeOnRootMismatch FSGroupChangePolicy = "OnRootMismatch" ) var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes" diff --git a/src/runtime/pkg/direct-volume/utils_test.go b/src/runtime/pkg/direct-volume/utils_test.go index 485f8f9ce..54f2ad388 100644 --- a/src/runtime/pkg/direct-volume/utils_test.go +++ b/src/runtime/pkg/direct-volume/utils_test.go @@ -25,7 +25,11 @@ func TestAdd(t *testing.T) { VolumeType: "block", Device: "/dev/sda", FsType: "ext4", - Options: []string{"journal_dev", "noload"}, + Metadata: map[string]string{ + FSGroupMetadataKey: "3000", + FSGroupChangePolicyMetadataKey: string(FSGroupChangeOnRootMismatch), + }, + Options: []string{"journal_dev", "noload"}, } buf, err := json.Marshal(actual) assert.Nil(t, err) @@ -39,6 +43,7 @@ func TestAdd(t *testing.T) { assert.Equal(t, expected.Device, actual.Device) assert.Equal(t, expected.FsType, actual.FsType) assert.Equal(t, expected.Options, actual.Options) + assert.Equal(t, expected.Metadata, actual.Metadata) // Remove the file err = Remove(volumePath) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index d82124c1d..4db6403ff 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -639,6 +639,26 @@ func (c *Container) createBlockDevices(ctx context.Context) error { c.mounts[i].Type = mntInfo.FsType c.mounts[i].Options = mntInfo.Options c.mounts[i].ReadOnly = readonly + + for key, value := range mntInfo.Metadata { + switch key { + case volume.FSGroupMetadataKey: + gid, err := strconv.Atoi(value) + if err != nil { + c.Logger().WithError(err).Errorf("invalid group id value %s provided for key %s", value, volume.FSGroupMetadataKey) + continue + } + c.mounts[i].FSGroup = &gid + case volume.FSGroupChangePolicyMetadataKey: + if _, exists := mntInfo.Metadata[volume.FSGroupMetadataKey]; !exists { + c.Logger().Errorf("%s specified without provding the group id with key %s", volume.FSGroupChangePolicyMetadataKey, volume.FSGroupMetadataKey) + continue + } + c.mounts[i].FSGroupChangePolicy = volume.FSGroupChangePolicy(value) + default: + c.Logger().Warnf("Ignoring unsupported direct-assignd volume metadata key: %s, value: %s", key, value) + } + } } var stat unix.Stat_t diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index aad6269d7..97a09d091 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -19,6 +19,7 @@ import ( "time" "github.com/docker/go-units" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" @@ -167,6 +168,15 @@ func getPagesizeFromOpt(fsOpts []string) string { return "" } +func getFSGroupChangePolicy(policy volume.FSGroupChangePolicy) pbTypes.FSGroupChangePolicy { + switch policy { + case volume.FSGroupChangeOnRootMismatch: + return pbTypes.FSGroupChangePolicy_OnRootMismatch + default: + return pbTypes.FSGroupChangePolicy_Always + } +} + // Shared path handling: // 1. create three directories for each sandbox: // -. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/, a directory to hold all host/guest shared mounts @@ -1468,6 +1478,12 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.De if len(vol.Options) == 0 { vol.Options = m.Options } + if m.FSGroup != nil { + vol.FsGroup = &grpc.FSGroup{ + GroupId: uint32(*m.FSGroup), + GroupChangePolicy: getFSGroupChangePolicy(m.FSGroupChangePolicy), + } + } return vol, nil } diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 029bc2a8d..af26edc0c 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "code.cloudfoundry.org/bytefmt" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" @@ -234,6 +235,7 @@ func TestHandleLocalStorage(t *testing.T) { } func TestHandleDeviceBlockVolume(t *testing.T) { + var gid = 2000 k := kataAgent{} // nolint: govet @@ -315,6 +317,27 @@ func TestHandleDeviceBlockVolume(t *testing.T) { Source: testSCSIAddr, }, }, + { + BlockDeviceDriver: config.VirtioBlock, + inputMount: Mount{ + FSGroup: &gid, + FSGroupChangePolicy: volume.FSGroupChangeOnRootMismatch, + }, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + PCIPath: testPCIPath, + VirtPath: testVirtPath, + }, + }, + resultVol: &pb.Storage{ + Driver: kataBlkDevType, + Source: testPCIPath.String(), + FsGroup: &pb.FSGroup{ + GroupId: uint32(gid), + GroupChangePolicy: pbTypes.FSGroupChangePolicy_OnRootMismatch, + }, + }, + }, } for _, test := range tests { diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index 0e83bc689..5e7582619 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -14,6 +14,7 @@ import ( "syscall" merr "github.com/hashicorp/go-multierror" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/pkg/errors" @@ -325,6 +326,7 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string } // Mount describes a container mount. +// nolint: govet type Mount struct { // Source is the source of the mount. Source string @@ -352,6 +354,14 @@ type Mount struct { // ReadOnly specifies if the mount should be read only or not ReadOnly bool + + // FSGroup a group ID that the group ownership of the files for the mounted volume + // will need to be changed when set. + FSGroup *int + + // FSGroupChangePolicy specifies the policy that will be used when applying + // group id ownership change for a volume. + FSGroupChangePolicy volume.FSGroupChangePolicy } func isSymlink(path string) bool { From 92c00c7e8429a6e6d056b26951a2e052e606cd15 Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Wed, 6 Apr 2022 19:33:05 -0700 Subject: [PATCH 04/15] agent: fsGroup support for direct-assigned volume Adding two functions set_ownership and recursive_ownership_change to support changing group id ownership for a mounted volume. The set_ownership will be called in common_storage_handler after mount_storage performs the mount for the volume. set_ownership will be a noop if the FSGroup field in the Storage struct is not set which indicates no chown will be performed. If FSGroup field is specified, then it will perform the recursive walk of the mounted volume path to change ownership of all files and directories to the desired group id. It will also configure the SetGid bit so that files created the directory will have group following parent directory group. If the fsGroupChangePolicy is on root mismatch, then the group ownership will be skipped if the root directory group id alreasy matches the desired group id and if the SetGid bit is also set on the root directory. This is the same behavior as what Kubelet does today when performing the recursive walk to change ownership. Fixes #4018 Signed-off-by: Yibo Zhuang --- src/agent/src/mount.rs | 311 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 307 insertions(+), 4 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 794724ba4..98cb6aeb8 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use tokio::sync::Mutex; use nix::mount::MsFlags; -use nix::unistd::Gid; +use nix::unistd::{Gid, Uid}; use regex::Regex; @@ -29,6 +29,7 @@ use crate::device::{ use crate::linux_abi::*; use crate::pci; use crate::protocols::agent::Storage; +use crate::protocols::types::FSGroupChangePolicy; use crate::Sandbox; #[cfg(target_arch = "s390x")] use crate::{ccw, device::get_virtio_blk_ccw_device_name}; @@ -43,6 +44,11 @@ pub const MOUNT_GUEST_TAG: &str = "kataShared"; // Allocating an FSGroup that owns the pod's volumes const FS_GID: &str = "fsgid"; +const RW_MASK: u32 = 0o660; +const RO_MASK: u32 = 0o440; +const EXEC_MASK: u32 = 0o110; +const MODE_SETGID: u32 = 0o2000; + #[rustfmt::skip] lazy_static! { pub static ref FLAGS: HashMap<&'static str, (bool, MsFlags)> = { @@ -222,7 +228,7 @@ async fn ephemeral_storage_handler( let meta = fs::metadata(&storage.mount_point)?; let mut permission = meta.permissions(); - let o_mode = meta.mode() | 0o2000; + let o_mode = meta.mode() | MODE_SETGID; permission.set_mode(o_mode); fs::set_permissions(&storage.mount_point, permission)?; } @@ -272,7 +278,7 @@ async fn local_storage_handler( if need_set_fsgid { // set SetGid mode mask. - o_mode |= 0o2000; + o_mode |= MODE_SETGID; } permission.set_mode(o_mode); @@ -489,7 +495,9 @@ fn common_storage_handler(logger: &Logger, storage: &Storage) -> Result // Mount the storage device. let mount_point = storage.mount_point.to_string(); - mount_storage(logger, storage).and(Ok(mount_point)) + mount_storage(logger, storage)?; + set_ownership(logger, storage)?; + Ok(mount_point) } // nvdimm_storage_handler handles the storage for NVDIMM driver. @@ -573,6 +581,91 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> { ) } +#[instrument] +pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> { + let logger = logger.new(o!("subsystem" => "mount", "fn" => "set_ownership")); + + // If fsGroup is not set, skip performing ownership change + if storage.fs_group.is_none() { + return Ok(()); + } + let fs_group = storage.get_fs_group(); + + let mut read_only = false; + let opts_vec: Vec = storage.options.to_vec(); + if opts_vec.contains(&String::from("ro")) { + read_only = true; + } + + let mount_path = Path::new(&storage.mount_point); + let metadata = mount_path.metadata().map_err(|err| { + error!(logger, "failed to obtain metadata for mount path"; + "mount-path" => mount_path.to_str(), + "error" => err.to_string(), + ); + err + })?; + + if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch + && metadata.gid() == fs_group.group_id + { + let mut mask = if read_only { RO_MASK } else { RW_MASK }; + mask |= EXEC_MASK; + + // With fsGroup change policy to OnRootMismatch, if the current + // gid of the mount path root directory matches the desired gid + // and the current permission of mount path root directory is correct, + // then ownership change will be skipped. + let current_mode = metadata.permissions().mode(); + if (mask & current_mode == mask) && (current_mode & MODE_SETGID != 0) { + info!(logger, "skipping ownership change for volume"; + "mount-path" => mount_path.to_str(), + "fs-group" => fs_group.group_id.to_string(), + ); + return Ok(()); + } + } + + info!(logger, "performing recursive ownership change"; + "mount-path" => mount_path.to_str(), + "fs-group" => fs_group.group_id.to_string(), + ); + recursive_ownership_change( + mount_path, + None, + Some(Gid::from_raw(fs_group.group_id)), + read_only, + ) +} + +#[instrument] +pub fn recursive_ownership_change( + path: &Path, + uid: Option, + gid: Option, + read_only: bool, +) -> Result<()> { + let mut mask = if read_only { RO_MASK } else { RW_MASK }; + if path.is_dir() { + for entry in fs::read_dir(&path)? { + recursive_ownership_change(entry?.path().as_path(), uid, gid, read_only)?; + } + mask |= EXEC_MASK; + mask |= MODE_SETGID; + } + nix::unistd::chown(path, uid, gid)?; + + if gid.is_some() { + let metadata = path.metadata()?; + let mut permission = metadata.permissions(); + let target_mode = metadata.mode() | mask; + permission.set_mode(target_mode); + fs::set_permissions(path, permission)?; + } + + Ok(()) +} + /// Looks for `mount_point` entry in the /proc/mounts. #[instrument] pub fn is_mounted(mount_point: &str) -> Result { @@ -925,6 +1018,8 @@ fn parse_options(option_list: Vec) -> HashMap { mod tests { use super::*; use crate::{skip_if_not_root, skip_loop_if_not_root, skip_loop_if_root}; + use protobuf::RepeatedField; + use protocols::agent::FSGroup; use std::fs::File; use std::fs::OpenOptions; use std::io::Write; @@ -1603,4 +1698,212 @@ mod tests { assert_eq!(expected_result, result, "{}", msg); } } + + #[test] + fn test_set_ownership() { + skip_if_not_root!(); + + let logger = slog::Logger::root(slog::Discard, o!()); + + #[derive(Debug)] + struct TestData<'a> { + mount_path: &'a str, + fs_group: Option, + read_only: bool, + expected_group_id: u32, + expected_permission: u32, + } + + let tests = &[ + TestData { + mount_path: "foo", + fs_group: None, + read_only: false, + expected_group_id: 0, + expected_permission: 0, + }, + TestData { + mount_path: "rw_mount", + fs_group: Some(FSGroup { + group_id: 3000, + group_change_policy: FSGroupChangePolicy::Always, + unknown_fields: Default::default(), + cached_size: Default::default(), + }), + read_only: false, + expected_group_id: 3000, + expected_permission: RW_MASK | EXEC_MASK | MODE_SETGID, + }, + TestData { + mount_path: "ro_mount", + fs_group: Some(FSGroup { + group_id: 3000, + group_change_policy: FSGroupChangePolicy::OnRootMismatch, + unknown_fields: Default::default(), + cached_size: Default::default(), + }), + read_only: true, + expected_group_id: 3000, + expected_permission: RO_MASK | EXEC_MASK | MODE_SETGID, + }, + ]; + + let tempdir = tempdir().expect("failed to create tmpdir"); + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let mount_dir = tempdir.path().join(d.mount_path); + fs::create_dir(&mount_dir) + .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg)); + + let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode(); + let mut storage_data = Storage::new(); + if d.read_only { + storage_data.set_options(RepeatedField::from_slice(&[ + "foo".to_string(), + "ro".to_string(), + ])); + } + if let Some(fs_group) = d.fs_group.clone() { + storage_data.set_fs_group(fs_group); + } + storage_data.mount_point = mount_dir.clone().into_os_string().into_string().unwrap(); + + let result = set_ownership(&logger, &storage_data); + assert!(result.is_ok()); + + assert_eq!( + mount_dir.as_path().metadata().unwrap().gid(), + d.expected_group_id + ); + assert_eq!( + mount_dir.as_path().metadata().unwrap().permissions().mode(), + (directory_mode | d.expected_permission) + ); + } + } + + #[test] + fn test_recursive_ownership_change() { + skip_if_not_root!(); + + const COUNT: usize = 5; + + #[derive(Debug)] + struct TestData<'a> { + // Directory where the recursive ownership change should be performed on + path: &'a str, + + // User ID for ownership change + uid: u32, + + // Group ID for ownership change + gid: u32, + + // Set when the permission should be read-only + read_only: bool, + + // The expected permission of all directories after ownership change + expected_permission_directory: u32, + + // The expected permission of all files after ownership change + expected_permission_file: u32, + } + + let tests = &[ + TestData { + path: "no_gid_change", + uid: 0, + gid: 0, + read_only: false, + expected_permission_directory: 0, + expected_permission_file: 0, + }, + TestData { + path: "rw_gid_change", + uid: 0, + gid: 3000, + read_only: false, + expected_permission_directory: RW_MASK | EXEC_MASK | MODE_SETGID, + expected_permission_file: RW_MASK, + }, + TestData { + path: "ro_gid_change", + uid: 0, + gid: 3000, + read_only: true, + expected_permission_directory: RO_MASK | EXEC_MASK | MODE_SETGID, + expected_permission_file: RO_MASK, + }, + ]; + + let tempdir = tempdir().expect("failed to create tmpdir"); + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let mount_dir = tempdir.path().join(d.path); + fs::create_dir(&mount_dir) + .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg)); + + let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode(); + let mut file_mode: u32 = 0; + + // create testing directories and files + for n in 1..COUNT { + let nest_dir = mount_dir.join(format!("nested{}", n)); + fs::create_dir(&nest_dir) + .unwrap_or_else(|_| panic!("{}: failed to create nest directory", msg)); + + for f in 1..COUNT { + let filename = nest_dir.join(format!("file{}", f)); + File::create(&filename) + .unwrap_or_else(|_| panic!("{}: failed to create file", msg)); + file_mode = filename.as_path().metadata().unwrap().permissions().mode(); + } + } + + let uid = if d.uid > 0 { + Some(Uid::from_raw(d.uid)) + } else { + None + }; + let gid = if d.gid > 0 { + Some(Gid::from_raw(d.gid)) + } else { + None + }; + let result = recursive_ownership_change(&mount_dir, uid, gid, d.read_only); + + assert!(result.is_ok()); + + assert_eq!(mount_dir.as_path().metadata().unwrap().gid(), d.gid); + assert_eq!( + mount_dir.as_path().metadata().unwrap().permissions().mode(), + (directory_mode | d.expected_permission_directory) + ); + + for n in 1..COUNT { + let nest_dir = mount_dir.join(format!("nested{}", n)); + for f in 1..COUNT { + let filename = nest_dir.join(format!("file{}", f)); + let file = Path::new(&filename); + + assert_eq!(file.metadata().unwrap().gid(), d.gid); + assert_eq!( + file.metadata().unwrap().permissions().mode(), + (file_mode | d.expected_permission_file) + ); + } + + let dir = Path::new(&nest_dir); + assert_eq!(dir.metadata().unwrap().gid(), d.gid); + assert_eq!( + dir.metadata().unwrap().permissions().mode(), + (directory_mode | d.expected_permission_directory) + ); + } + } + } } From 9c22d9554e35712e3a5e404ca20b118645b3ad64 Mon Sep 17 00:00:00 2001 From: Braden Rayhorn Date: Sun, 3 Apr 2022 17:15:12 -0500 Subject: [PATCH 05/15] agent: add tests for update_container_namespaces Add test coverage for update_container_namespaces function in src/rpc.rs. Includes minor refactor to make function easier to test. Fixes #4034 Signed-off-by: Braden Rayhorn --- src/agent/src/rpc.rs | 143 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 139 insertions(+), 4 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 50a11dfd9..310746c23 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -87,6 +87,8 @@ const MODPROBE_PATH: &str = "/sbin/modprobe"; const ERR_CANNOT_GET_WRITER: &str = "Cannot get writer"; const ERR_INVALID_BLOCK_SIZE: &str = "Invalid block size"; +const ERR_NO_LINUX_FIELD: &str = "Spec does not contain linux field"; +const ERR_NO_SANDBOX_PIDNS: &str = "Sandbox does not have sandbox_pidns"; // Convenience macro to obtain the scope logger macro_rules! sl { @@ -1588,7 +1590,7 @@ fn update_container_namespaces( let linux = spec .linux .as_mut() - .ok_or_else(|| anyhow!("Spec didn't container linux field"))?; + .ok_or_else(|| anyhow!(ERR_NO_LINUX_FIELD))?; let namespaces = linux.namespaces.as_mut_slice(); for namespace in namespaces.iter_mut() { @@ -1615,7 +1617,7 @@ fn update_container_namespaces( if let Some(ref pidns) = &sandbox.sandbox_pidns { pid_ns.path = String::from(pidns.path.as_str()); } else { - return Err(anyhow!("failed to get sandbox pidns")); + return Err(anyhow!(ERR_NO_SANDBOX_PIDNS)); } } @@ -1879,8 +1881,8 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::protocols::agent_ttrpc::AgentService as _; - use oci::{Hook, Hooks}; + use crate::{namespace::Namespace, protocols::agent_ttrpc::AgentService as _}; + use oci::{Hook, Hooks, Linux, LinuxNamespace}; use tempfile::{tempdir, TempDir}; use ttrpc::{r#async::TtrpcContext, MessageHeader}; @@ -2192,6 +2194,139 @@ mod tests { } } + #[tokio::test] + async fn test_update_container_namespaces() { + #[derive(Debug)] + struct TestData<'a> { + has_linux_in_spec: bool, + sandbox_pidns_path: Option<&'a str>, + + namespaces: Vec, + use_sandbox_pidns: bool, + result: Result<()>, + expected_namespaces: Vec, + } + + impl Default for TestData<'_> { + fn default() -> Self { + TestData { + has_linux_in_spec: true, + sandbox_pidns_path: Some("sharedpidns"), + namespaces: vec![ + LinuxNamespace { + r#type: NSTYPEIPC.to_string(), + path: "ipcpath".to_string(), + }, + LinuxNamespace { + r#type: NSTYPEUTS.to_string(), + path: "utspath".to_string(), + }, + ], + use_sandbox_pidns: false, + result: Ok(()), + expected_namespaces: vec![ + LinuxNamespace { + r#type: NSTYPEIPC.to_string(), + path: "".to_string(), + }, + LinuxNamespace { + r#type: NSTYPEUTS.to_string(), + path: "".to_string(), + }, + LinuxNamespace { + r#type: NSTYPEPID.to_string(), + path: "".to_string(), + }, + ], + } + } + } + + let tests = &[ + TestData { + ..Default::default() + }, + TestData { + use_sandbox_pidns: true, + expected_namespaces: vec![ + LinuxNamespace { + r#type: NSTYPEIPC.to_string(), + path: "".to_string(), + }, + LinuxNamespace { + r#type: NSTYPEUTS.to_string(), + path: "".to_string(), + }, + LinuxNamespace { + r#type: NSTYPEPID.to_string(), + path: "sharedpidns".to_string(), + }, + ], + ..Default::default() + }, + TestData { + namespaces: vec![], + use_sandbox_pidns: true, + expected_namespaces: vec![LinuxNamespace { + r#type: NSTYPEPID.to_string(), + path: "sharedpidns".to_string(), + }], + ..Default::default() + }, + TestData { + namespaces: vec![], + use_sandbox_pidns: false, + expected_namespaces: vec![LinuxNamespace { + r#type: NSTYPEPID.to_string(), + path: "".to_string(), + }], + ..Default::default() + }, + TestData { + namespaces: vec![], + sandbox_pidns_path: None, + use_sandbox_pidns: true, + result: Err(anyhow!(ERR_NO_SANDBOX_PIDNS)), + expected_namespaces: vec![], + ..Default::default() + }, + TestData { + has_linux_in_spec: false, + result: Err(anyhow!(ERR_NO_LINUX_FIELD)), + ..Default::default() + }, + ]; + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let logger = slog::Logger::root(slog::Discard, o!()); + let mut sandbox = Sandbox::new(&logger).unwrap(); + if let Some(pidns_path) = d.sandbox_pidns_path { + let mut sandbox_pidns = Namespace::new(&logger); + sandbox_pidns.path = pidns_path.to_string(); + sandbox.sandbox_pidns = Some(sandbox_pidns); + } + + let mut oci = Spec::default(); + if d.has_linux_in_spec { + oci.linux = Some(Linux { + namespaces: d.namespaces.clone(), + ..Default::default() + }); + } + + let result = update_container_namespaces(&sandbox, &mut oci, d.use_sandbox_pidns); + + let msg = format!("{}, result: {:?}", msg, result); + + assert_result!(d.result, result, msg); + if let Some(linux) = oci.linux { + assert_eq!(d.expected_namespaces, linux.namespaces, "{}", msg); + } + } + } + #[tokio::test] async fn test_get_memory_info() { #[derive(Debug)] From c3776b17923b425f6bc89cff02f01bf7af40932d Mon Sep 17 00:00:00 2001 From: Braden Rayhorn Date: Tue, 22 Mar 2022 22:30:18 -0500 Subject: [PATCH 06/15] agent: add tests for is_signal_handled function Add test coverage for is_signal_handled function in rpc.rs. Includes refactors to make the function testable and handle additional cases. Fixes #3939 Signed-off-by: Braden Rayhorn --- src/agent/src/rpc.rs | 129 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 119 insertions(+), 10 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 50a11dfd9..051848b19 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -407,7 +407,8 @@ impl AgentService { // For container initProcess, if it hasn't installed handler for "SIGTERM" signal, // it will ignore the "SIGTERM" signal sent to it, thus send it "SIGKILL" signal // instead of "SIGTERM" to terminate it. - if p.init && sig == libc::SIGTERM && !is_signal_handled(p.pid, sig as u32) { + let proc_status_file = format!("/proc/{}/status", p.pid); + if p.init && sig == libc::SIGTERM && !is_signal_handled(&proc_status_file, sig as u32) { sig = libc::SIGKILL; } p.signal(sig)?; @@ -1635,21 +1636,33 @@ fn append_guest_hooks(s: &Sandbox, oci: &mut Spec) -> Result<()> { Ok(()) } -// Check is the container process installed the +// Check if the container process installed the // handler for specific signal. -fn is_signal_handled(pid: pid_t, signum: u32) -> bool { - let sig_mask: u64 = 1u64 << (signum - 1); - let file_name = format!("/proc/{}/status", pid); +fn is_signal_handled(proc_status_file: &str, signum: u32) -> bool { + let shift_count: u64 = if signum == 0 { + // signum 0 is used to check for process liveness. + // Since that signal is not part of the mask in the file, we only need + // to know if the file (and therefore) process exists to handle + // that signal. + return fs::metadata(proc_status_file).is_ok(); + } else if signum > 64 { + // Ensure invalid signum won't break bit shift logic + warn!(sl!(), "received invalid signum {}", signum); + return false; + } else { + (signum - 1).into() + }; // Open the file in read-only mode (ignoring errors). - let file = match File::open(&file_name) { + let file = match File::open(proc_status_file) { Ok(f) => f, Err(_) => { - warn!(sl!(), "failed to open file {}\n", file_name); + warn!(sl!(), "failed to open file {}", proc_status_file); return false; } }; + let sig_mask: u64 = 1 << shift_count; let reader = BufReader::new(file); // Read the file line by line using the lines() iterator from std::io::BufRead. @@ -1657,21 +1670,21 @@ fn is_signal_handled(pid: pid_t, signum: u32) -> bool { let line = match line { Ok(l) => l, Err(_) => { - warn!(sl!(), "failed to read file {}\n", file_name); + warn!(sl!(), "failed to read file {}", proc_status_file); return false; } }; if line.starts_with("SigCgt:") { let mask_vec: Vec<&str> = line.split(':').collect(); if mask_vec.len() != 2 { - warn!(sl!(), "parse the SigCgt field failed\n"); + warn!(sl!(), "parse the SigCgt field failed"); return false; } let sig_cgt_str = mask_vec[1]; let sig_cgt_mask = match u64::from_str_radix(sig_cgt_str, 16) { Ok(h) => h, Err(_) => { - warn!(sl!(), "failed to parse the str {} to hex\n", sig_cgt_str); + warn!(sl!(), "failed to parse the str {} to hex", sig_cgt_str); return false; } }; @@ -2305,6 +2318,102 @@ mod tests { } } + #[tokio::test] + async fn test_is_signal_handled() { + #[derive(Debug)] + struct TestData<'a> { + status_file_data: Option<&'a str>, + signum: u32, + result: bool, + } + + let tests = &[ + TestData { + status_file_data: Some( + r#" +SigBlk:0000000000010000 +SigCgt:0000000000000001 +OtherField:other + "#, + ), + signum: 1, + result: true, + }, + TestData { + status_file_data: Some("SigCgt:000000004b813efb"), + signum: 4, + result: true, + }, + TestData { + status_file_data: Some("SigCgt:000000004b813efb"), + signum: 3, + result: false, + }, + TestData { + status_file_data: Some("SigCgt:000000004b813efb"), + signum: 65, + result: false, + }, + TestData { + status_file_data: Some("SigCgt:000000004b813efb"), + signum: 0, + result: true, + }, + TestData { + status_file_data: Some("SigCgt:ZZZZZZZZ"), + signum: 1, + result: false, + }, + TestData { + status_file_data: Some("SigCgt:-1"), + signum: 1, + result: false, + }, + TestData { + status_file_data: Some("SigCgt"), + signum: 1, + result: false, + }, + TestData { + status_file_data: Some("any data"), + signum: 0, + result: true, + }, + TestData { + status_file_data: Some("SigBlk:0000000000000001"), + signum: 1, + result: false, + }, + TestData { + status_file_data: None, + signum: 1, + result: false, + }, + TestData { + status_file_data: None, + signum: 0, + result: false, + }, + ]; + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let dir = tempdir().expect("failed to make tempdir"); + let proc_status_file_path = dir.path().join("status"); + + if let Some(file_data) = d.status_file_data { + fs::write(&proc_status_file_path, file_data).unwrap(); + } + + let result = is_signal_handled(proc_status_file_path.to_str().unwrap(), d.signum); + + let msg = format!("{}, result: {:?}", msg, result); + + assert_eq!(d.result, result, "{}", msg); + } + } + #[tokio::test] async fn test_verify_cid() { #[derive(Debug)] From 9b6f24b2ee8d9f9cc7ac3c053716f73d15d12594 Mon Sep 17 00:00:00 2001 From: Braden Rayhorn Date: Mon, 11 Apr 2022 21:08:34 -0500 Subject: [PATCH 07/15] agent: add tests for mount_to_rootfs function Add test coverage for mount_to_rootfs function in src/mount.rs. Includes minor refactoring to make function more easily testable. Fixes #4073 Signed-off-by: Braden Rayhorn --- src/agent/src/mount.rs | 117 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 7 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index da46ddae6..3e976ec8a 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -85,11 +85,11 @@ lazy_static! { } #[derive(Debug, PartialEq)] -pub struct InitMount { - fstype: &'static str, - src: &'static str, - dest: &'static str, - options: Vec<&'static str>, +pub struct InitMount<'a> { + fstype: &'a str, + src: &'a str, + dest: &'a str, + options: Vec<&'a str>, } #[rustfmt::skip] @@ -115,7 +115,7 @@ lazy_static!{ #[rustfmt::skip] lazy_static! { - pub static ref INIT_ROOTFS_MOUNTS: Vec = vec![ + pub static ref INIT_ROOTFS_MOUNTS: Vec> = vec![ InitMount{fstype: "proc", src: "proc", dest: "/proc", options: vec!["nosuid", "nodev", "noexec"]}, InitMount{fstype: "sysfs", src: "sysfs", dest: "/sys", options: vec!["nosuid", "nodev", "noexec"]}, InitMount{fstype: "devtmpfs", src: "dev", dest: "/dev", options: vec!["nosuid"]}, @@ -776,7 +776,7 @@ pub fn get_cgroup_mounts( logger: &Logger, cg_path: &str, unified_cgroup_hierarchy: bool, -) -> Result> { +) -> Result>> { // cgroup v2 // https://github.com/kata-containers/agent/blob/8c9bbadcd448c9a67690fbe11a860aaacc69813c/agent.go#L1249 if unified_cgroup_hierarchy { @@ -1621,6 +1621,109 @@ mod tests { } } + #[test] + fn test_mount_to_rootfs() { + #[derive(Debug)] + struct TestData<'a> { + test_user: TestUserType, + src: &'a str, + options: Vec<&'a str>, + error_contains: &'a str, + deny_mount_dir_permission: bool, + // if true src will be prepended with a temporary directory + mask_src: bool, + } + + impl Default for TestData<'_> { + fn default() -> Self { + TestData { + test_user: TestUserType::Any, + src: "src", + options: vec![], + error_contains: "", + deny_mount_dir_permission: false, + mask_src: true, + } + } + } + + let tests = &[ + TestData { + test_user: TestUserType::NonRootOnly, + error_contains: "EPERM: Operation not permitted", + ..Default::default() + }, + TestData { + test_user: TestUserType::NonRootOnly, + src: "dev", + mask_src: false, + ..Default::default() + }, + TestData { + test_user: TestUserType::RootOnly, + ..Default::default() + }, + TestData { + test_user: TestUserType::NonRootOnly, + deny_mount_dir_permission: true, + error_contains: "could not create directory", + ..Default::default() + }, + ]; + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + if d.test_user == TestUserType::RootOnly { + skip_loop_if_not_root!(msg); + } else if d.test_user == TestUserType::NonRootOnly { + skip_loop_if_root!(msg); + } + + let drain = slog::Discard; + let logger = slog::Logger::root(drain, o!()); + let tempdir = tempdir().unwrap(); + + let src = if d.mask_src { + tempdir.path().join(&d.src) + } else { + Path::new(d.src).to_path_buf() + }; + let dest = tempdir.path().join("mnt"); + let init_mount = InitMount { + fstype: "tmpfs", + src: src.to_str().unwrap(), + dest: dest.to_str().unwrap(), + options: d.options.clone(), + }; + + if d.deny_mount_dir_permission { + fs::set_permissions(dest.parent().unwrap(), fs::Permissions::from_mode(0o000)) + .unwrap(); + } + + let result = mount_to_rootfs(&logger, &init_mount); + + // restore permissions so tempdir can be cleaned up + if d.deny_mount_dir_permission { + fs::set_permissions(dest.parent().unwrap(), fs::Permissions::from_mode(0o755)) + .unwrap(); + } + + if result.is_ok() && d.mask_src { + nix::mount::umount(&dest).unwrap(); + } + + let msg = format!("{}: result: {:?}", msg, result); + if d.error_contains.is_empty() { + assert!(result.is_ok(), "{}", msg); + } else { + assert!(result.is_err(), "{}", msg); + let error_msg = format!("{}", result.unwrap_err()); + assert!(error_msg.contains(d.error_contains), "{}", msg); + } + } + } + #[test] fn test_get_pagesize_and_size_from_option() { let expected_pagesize = 2048; From 6e79042aa06197e186ad9ba65d3c23522dce4944 Mon Sep 17 00:00:00 2001 From: Zhuoyu Tie Date: Fri, 8 Apr 2022 17:06:08 +0800 Subject: [PATCH 08/15] runtime: no need to write virtiofsd error to log The scanner reads nothing from viriofsd stderr pipe, because param '--syslog' rediercts stderr to syslog. So there is no need to write scanner.Text() to kata log Fixes: #4063 Signed-off-by: Zhuoyu Tie --- src/runtime/virtcontainers/virtiofsd.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go index f9c567ce0..0d8ba1c19 100644 --- a/src/runtime/virtcontainers/virtiofsd.go +++ b/src/runtime/virtcontainers/virtiofsd.go @@ -6,7 +6,6 @@ package virtcontainers import ( - "bufio" "context" "fmt" "net" @@ -134,24 +133,14 @@ func (v *virtiofsd) Start(ctx context.Context, onQuit onQuitFunc) (int, error) { v.Logger().WithField("path", v.path).Info() v.Logger().WithField("args", strings.Join(args, " ")).Info() - stderr, err := cmd.StderrPipe() - if err != nil { - return pid, err - } if err = utils.StartCmd(cmd); err != nil { return pid, err } - // Monitor virtiofsd's stderr and stop sandbox if virtiofsd quits go func() { - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - v.Logger().WithField("source", "virtiofsd").Info(scanner.Text()) - } - v.Logger().Info("virtiofsd quits") - // Wait to release resources of virtiofsd process cmd.Process.Wait() + v.Logger().Info("virtiofsd quits") if onQuit != nil { onQuit() } From 86977ff7809ce59c66fe7c7bf506a8bbf4b77a20 Mon Sep 17 00:00:00 2001 From: Francesco Giudici Date: Thu, 7 Apr 2022 11:18:46 +0200 Subject: [PATCH 09/15] kata-monitor: update the hrefs in the debug/pprof index page kata-monitor allows to get data profiles from the kata shim instances running on the same node by acting as a proxy (e.g., http://$NODE_ADDRESS:8090/debug/pprof/?sandbox=$MYSANDBOXID). In order to proxy the requests and the responses to the right shim, kata-monitor requires to pass the sandbox id via a query string in the url. The profiling index page proxied by kata-monitor contains the link to all the data profiles available. All the links anyway do not contain the sandbox id included in the request: the links result then broken when accessed through kata-monitor. This happens because the profiling index page comes from the kata shim, which will not include the query string provided in the http request. Let's add on-the-fly the sandbox id in each href tag returned by the kata shim index page before providing the proxied page. Fixes: #4054 Signed-off-by: Francesco Giudici --- src/runtime/pkg/kata-monitor/pprof.go | 54 ++++++++-- src/runtime/pkg/kata-monitor/pprof_test.go | 117 +++++++++++++++++++++ 2 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 src/runtime/pkg/kata-monitor/pprof_test.go diff --git a/src/runtime/pkg/kata-monitor/pprof.go b/src/runtime/pkg/kata-monitor/pprof.go index 5dac2da03..62ed70c2e 100644 --- a/src/runtime/pkg/kata-monitor/pprof.go +++ b/src/runtime/pkg/kata-monitor/pprof.go @@ -10,6 +10,8 @@ import ( "io" "net" "net/http" + "regexp" + "strings" cdshim "github.com/containerd/containerd/runtime/v2/shim" @@ -33,7 +35,13 @@ func (km *KataMonitor) composeSocketAddress(r *http.Request) (string, error) { return shim.SocketAddress(sandbox), nil } -func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request) { +func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request, + proxyResponse func(req *http.Request, w io.Writer, r io.Reader) error) { + + if proxyResponse == nil { + proxyResponse = copyResponse + } + w.Header().Set("X-Content-Type-Options", "nosniff") socketAddress, err := km.composeSocketAddress(r) @@ -73,38 +81,68 @@ func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Disposition", contentDisposition) } - io.Copy(w, output) + err = proxyResponse(r, w, output) + if err != nil { + monitorLog.WithError(err).Errorf("failed proxying %s from %s", uri, socketAddress) + serveError(w, http.StatusInternalServerError, "error retrieving resource") + } } // ExpvarHandler handles other `/debug/vars` requests func (km *KataMonitor) ExpvarHandler(w http.ResponseWriter, r *http.Request) { - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) } // PprofIndex handles other `/debug/pprof/` requests func (km *KataMonitor) PprofIndex(w http.ResponseWriter, r *http.Request) { - km.proxyRequest(w, r) + if len(strings.TrimPrefix(r.URL.Path, "/debug/pprof/")) == 0 { + km.proxyRequest(w, r, copyResponseAddingSandboxIdToHref) + } else { + km.proxyRequest(w, r, nil) + } } // PprofCmdline handles other `/debug/cmdline` requests func (km *KataMonitor) PprofCmdline(w http.ResponseWriter, r *http.Request) { - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) } // PprofProfile handles other `/debug/profile` requests func (km *KataMonitor) PprofProfile(w http.ResponseWriter, r *http.Request) { - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) } // PprofSymbol handles other `/debug/symbol` requests func (km *KataMonitor) PprofSymbol(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) } // PprofTrace handles other `/debug/trace` requests func (km *KataMonitor) PprofTrace(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", `attachment; filename="trace"`) - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) +} + +func copyResponse(req *http.Request, w io.Writer, r io.Reader) error { + _, err := io.Copy(w, r) + return err +} + +func copyResponseAddingSandboxIdToHref(req *http.Request, w io.Writer, r io.Reader) error { + sb, err := getSandboxIDFromReq(req) + if err != nil { + monitorLog.WithError(err).Warning("missing sandbox query in pprof url") + return copyResponse(req, w, r) + } + buf, err := io.ReadAll(r) + if err != nil { + return err + } + + re := regexp.MustCompile(``) + outHtml := re.ReplaceAllString(string(buf), fmt.Sprintf("", sb)) + w.Write([]byte(outHtml)) + return nil } diff --git a/src/runtime/pkg/kata-monitor/pprof_test.go b/src/runtime/pkg/kata-monitor/pprof_test.go new file mode 100644 index 000000000..e02dc00b1 --- /dev/null +++ b/src/runtime/pkg/kata-monitor/pprof_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2022 Red Hat Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katamonitor + +import ( + "bytes" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCopyResponseAddingSandboxIdToHref(t *testing.T) { + assert := assert.New(t) + + htmlIn := strings.NewReader(` + + +/debug/pprof/ + + + +/debug/pprof/
+
+Types of profiles available: + + + + + + + + + + + +
CountProfile
27allocs
0block
0cmdline
39goroutine
27heap
0mutex
0profile
10threadcreate
0trace
+full goroutine stack dump +
+

+Profile Descriptions: +

    +
  • allocs:
    A sampling of all past memory allocations
    • +
  • block:
    Stack traces that led to blocking on synchronization primitives
    • +
  • cmdline:
    The command line invocation of the current program
    • +
  • goroutine:
    Stack traces of all current goroutines
    • +
  • heap:
    A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
    • +
  • mutex:
    Stack traces of holders of contended mutexes
    • +
  • profile:
    CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.
    • +
  • threadcreate:
    Stack traces that led to the creation of new OS threads
    • +
  • trace:
    A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.
    • +
+

+ +`) + + htmlExpected := bytes.NewBufferString(` + + +/debug/pprof/ + + + +/debug/pprof/
+
+Types of profiles available: + + + + + + + + + + + +
CountProfile
27allocs
0block
0cmdline
39goroutine
27heap
0mutex
0profile
10threadcreate
0trace
+full goroutine stack dump +
+

+Profile Descriptions: +

    +
  • allocs:
    A sampling of all past memory allocations
    • +
  • block:
    Stack traces that led to blocking on synchronization primitives
    • +
  • cmdline:
    The command line invocation of the current program
    • +
  • goroutine:
    Stack traces of all current goroutines
    • +
  • heap:
    A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
    • +
  • mutex:
    Stack traces of holders of contended mutexes
    • +
  • profile:
    CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.
    • +
  • threadcreate:
    Stack traces that led to the creation of new OS threads
    • +
  • trace:
    A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.
    • +
+

+ +`) + + req := &http.Request{URL: &url.URL{RawQuery: "sandbox=1234567890"}} + buf := bytes.NewBuffer(nil) + copyResponseAddingSandboxIdToHref(req, buf, htmlIn) + assert.Equal(htmlExpected, buf) +} From d136c9c24095599920e6312d79265e2e1c6ffdf6 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Wed, 13 Apr 2022 08:54:01 +0200 Subject: [PATCH 10/15] test: Fix golangci-lint error for s390x This is to fix a test failure for the kata-containers-2.0-ubuntu-20.04-s390x-main-baseline jenkins job Fixes: #4088 Signed-off-by: Hyounggyu Choi --- src/runtime/cmd/kata-runtime/kata-check_s390x_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/cmd/kata-runtime/kata-check_s390x_test.go b/src/runtime/cmd/kata-runtime/kata-check_s390x_test.go index 0898037c6..a91d7d66c 100644 --- a/src/runtime/cmd/kata-runtime/kata-check_s390x_test.go +++ b/src/runtime/cmd/kata-runtime/kata-check_s390x_test.go @@ -126,7 +126,7 @@ func TestKvmIsUsable(t *testing.T) { kvmDevice = savedKvmDevice }() - err = kvmIsUsable() + err := kvmIsUsable() assert.Error(err) err = createEmptyFile(fakeKVMDevice) From aabcebbf58ea2d4a3ec1e681e19b236fefa38088 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Mon, 11 Apr 2022 18:44:23 -0700 Subject: [PATCH 11/15] agent: best-effort removing mount point During container exit, the agent tries to remove all the mount point directories, which can fail if it's a readonly filesytem (e.g. device mapper). This commit ignores the removal failure and logs a warning message. Fixes: #4043 Signed-off-by: Feng Wang --- src/agent/src/sandbox.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index 3577f7f3f..84cc65929 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -151,7 +151,12 @@ impl Sandbox { pub fn remove_sandbox_storage(&self, path: &str) -> Result<()> { let mounts = vec![path.to_string()]; remove_mounts(&mounts)?; - fs::remove_dir_all(path).context(format!("failed to remove dir {:?}", path))?; + // "remove_dir" will fail if the mount point is backed by a read-only filesystem. + // This is the case with the device mapper snapshotter, where we mount the block device directly + // at the underlying sandbox path which was provided from the base RO kataShared path from the host. + if let Err(err) = fs::remove_dir(path) { + warn!(self.logger, "failed to remove dir {}, {:?}", path, err); + } Ok(()) } @@ -564,19 +569,8 @@ mod tests { .remove_sandbox_storage(invalid_dir.to_str().unwrap()) .is_err()); - // Now, create a double mount as this guarantees the directory cannot - // be deleted after the first umount. - for _i in 0..2 { - assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok()); - } + assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok()); - assert!( - s.remove_sandbox_storage(destdir_path).is_err(), - "Expect fail as deletion cannot happen due to the second mount." - ); - - // This time it should work as the previous two calls have undone the double - // mount. assert!(s.remove_sandbox_storage(destdir_path).is_ok()); } From 6012c19707ed85cc3abdcef92f22340161ff8520 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Thu, 14 Apr 2022 12:52:35 -0700 Subject: [PATCH 12/15] versions: Upgrade to Cloud Hypervisor v23.0 Highlights from the Cloud Hypervisor release v23.0: 1) vDPA Support; 2) Updated OS Support list (Jammy 22.04 added with EOLed versions removed); 3) AArch64 Memory Map Improvements; 4) AMX Support; 5) Bug Fixes; Details can be found: https://github.com/cloud-hypervisor/cloud-hypervisor/releases/tag/v23.0 Fixes: #4101 Signed-off-by: Bo Chen --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index afd8b27f9..c885b00c9 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "v22.1" + version: "v23.0" firecracker: description: "Firecracker micro-VMM" From 29e569aa92a13f37db5646f891c5baa45d2bc722 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Thu, 14 Apr 2022 12:56:01 -0700 Subject: [PATCH 13/15] virtcontainers: clh: Re-generate the client code This patch re-generates the client code for Cloud Hypervisor v23.0. Note: The client code of cloud-hypervisor's (CLH) OpenAPI is automatically generated by openapi-generator [1-2]. [1] https://github.com/OpenAPITools/openapi-generator [2] https://github.com/kata-containers/kata-containers/blob/main/src/runtime/virtcontainers/pkg/cloud-hypervisor/README.md Signed-off-by: Bo Chen --- .../client/.openapi-generator/FILES | 4 + .../pkg/cloud-hypervisor/client/README.md | 3 + .../cloud-hypervisor/client/api/openapi.yaml | 192 ++++++++++---- .../cloud-hypervisor/client/api_default.go | 111 ++++++++ .../client/docs/CpuFeatures.md | 56 ++++ .../client/docs/CpusConfig.md | 26 ++ .../client/docs/DefaultApi.md | 65 +++++ .../client/docs/VdpaConfig.md | 150 +++++++++++ .../cloud-hypervisor/client/docs/VmConfig.md | 26 ++ .../client/model_cpu_features.go | 113 ++++++++ .../client/model_cpus_config.go | 36 +++ .../client/model_vdpa_config.go | 249 ++++++++++++++++++ .../client/model_vm_config.go | 36 +++ .../cloud-hypervisor/cloud-hypervisor.yaml | 55 +++- 14 files changed, 1069 insertions(+), 53 deletions(-) create mode 100644 src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpuFeatures.md create mode 100644 src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VdpaConfig.md create mode 100644 src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpu_features.go create mode 100644 src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vdpa_config.go diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES index ed5dc5764..7eb679fb2 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/.openapi-generator/FILES @@ -10,6 +10,7 @@ docs/BalloonConfig.md docs/CmdLineConfig.md docs/ConsoleConfig.md docs/CpuAffinity.md +docs/CpuFeatures.md docs/CpuTopology.md docs/CpusConfig.md docs/DefaultApi.md @@ -35,6 +36,7 @@ docs/SendMigrationData.md docs/SgxEpcConfig.md docs/TdxConfig.md docs/TokenBucket.md +docs/VdpaConfig.md docs/VmAddDevice.md docs/VmConfig.md docs/VmInfo.md @@ -51,6 +53,7 @@ model_balloon_config.go model_cmd_line_config.go model_console_config.go model_cpu_affinity.go +model_cpu_features.go model_cpu_topology.go model_cpus_config.go model_device_config.go @@ -75,6 +78,7 @@ model_send_migration_data.go model_sgx_epc_config.go model_tdx_config.go model_token_bucket.go +model_vdpa_config.go model_vm_add_device.go model_vm_config.go model_vm_info.go diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md index 3b7967c4b..2e12b424c 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/README.md @@ -92,6 +92,7 @@ Class | Method | HTTP request | Description *DefaultApi* | [**VmAddFsPut**](docs/DefaultApi.md#vmaddfsput) | **Put** /vm.add-fs | Add a new virtio-fs device to the VM *DefaultApi* | [**VmAddNetPut**](docs/DefaultApi.md#vmaddnetput) | **Put** /vm.add-net | Add a new network device to the VM *DefaultApi* | [**VmAddPmemPut**](docs/DefaultApi.md#vmaddpmemput) | **Put** /vm.add-pmem | Add a new pmem device to the VM +*DefaultApi* | [**VmAddVdpaPut**](docs/DefaultApi.md#vmaddvdpaput) | **Put** /vm.add-vdpa | Add a new vDPA device to the VM *DefaultApi* | [**VmAddVsockPut**](docs/DefaultApi.md#vmaddvsockput) | **Put** /vm.add-vsock | Add a new vsock device to the VM *DefaultApi* | [**VmCountersGet**](docs/DefaultApi.md#vmcountersget) | **Get** /vm.counters | Get counters from the VM *DefaultApi* | [**VmInfoGet**](docs/DefaultApi.md#vminfoget) | **Get** /vm.info | Returns general information about the cloud-hypervisor Virtual Machine (VM) instance. @@ -111,6 +112,7 @@ Class | Method | HTTP request | Description - [CmdLineConfig](docs/CmdLineConfig.md) - [ConsoleConfig](docs/ConsoleConfig.md) - [CpuAffinity](docs/CpuAffinity.md) + - [CpuFeatures](docs/CpuFeatures.md) - [CpuTopology](docs/CpuTopology.md) - [CpusConfig](docs/CpusConfig.md) - [DeviceConfig](docs/DeviceConfig.md) @@ -135,6 +137,7 @@ Class | Method | HTTP request | Description - [SgxEpcConfig](docs/SgxEpcConfig.md) - [TdxConfig](docs/TdxConfig.md) - [TokenBucket](docs/TokenBucket.md) + - [VdpaConfig](docs/VdpaConfig.md) - [VmAddDevice](docs/VmAddDevice.md) - [VmConfig](docs/VmConfig.md) - [VmInfo](docs/VmInfo.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml index 4a168e53e..43b2e9737 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api/openapi.yaml @@ -306,6 +306,28 @@ paths: "500": description: The new device could not be added to the VM instance. summary: Add a new vsock device to the VM + /vm.add-vdpa: + put: + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/VdpaConfig' + description: The details of the new vDPA device + required: true + responses: + "200": + content: + application/json: + schema: + $ref: '#/components/schemas/PciDeviceInfo' + description: The new vDPA device was successfully added to the VM instance. + "204": + description: The new vDPA device was successfully (cold) added to the VM + instance. + "500": + description: The new vDPA device could not be added to the VM instance. + summary: Add a new vDPA device to the VM /vm.snapshot: put: requestBody: @@ -386,7 +408,7 @@ components: VmInfo: description: Virtual Machine information example: - memory_actual_size: 3 + memory_actual_size: 7 state: Created config: console: @@ -472,6 +494,8 @@ components: refill_time: 0 id: id cpus: + features: + amx: true topology: dies_per_package: 5 threads_per_core: 1 @@ -500,37 +524,48 @@ components: id: id kernel: path: path + vdpa: + - pci_segment: 7 + path: path + num_queues: 3 + iommu: false + id: id + - pci_segment: 7 + path: path + num_queues: 3 + iommu: false + id: id numa: - distances: - - distance: 4 - destination: 0 - - distance: 4 - destination: 0 + - distance: 7 + destination: 8 + - distance: 7 + destination: 8 cpus: - - 6 - - 6 + - 4 + - 4 sgx_epc_sections: - sgx_epc_sections - sgx_epc_sections memory_zones: - memory_zones - memory_zones - guest_numa_id: 7 + guest_numa_id: 0 - distances: - - distance: 4 - destination: 0 - - distance: 4 - destination: 0 + - distance: 7 + destination: 8 + - distance: 7 + destination: 8 cpus: - - 6 - - 6 + - 4 + - 4 sgx_epc_sections: - sgx_epc_sections - sgx_epc_sections memory_zones: - memory_zones - memory_zones - guest_numa_id: 7 + guest_numa_id: 0 tdx: firmware: firmware rng: @@ -538,10 +573,10 @@ components: src: /dev/urandom sgx_epc: - prefault: false - size: 0 + size: 6 id: id - prefault: false - size: 0 + size: 6 id: id fs: - pci_segment: 6 @@ -568,9 +603,9 @@ components: cid: 3 platform: iommu_segments: - - 7 - - 7 - num_pci_segments: 8 + - 3 + - 3 + num_pci_segments: 3 pmem: - pci_segment: 6 mergeable: false @@ -801,6 +836,8 @@ components: refill_time: 0 id: id cpus: + features: + amx: true topology: dies_per_package: 5 threads_per_core: 1 @@ -829,37 +866,48 @@ components: id: id kernel: path: path + vdpa: + - pci_segment: 7 + path: path + num_queues: 3 + iommu: false + id: id + - pci_segment: 7 + path: path + num_queues: 3 + iommu: false + id: id numa: - distances: - - distance: 4 - destination: 0 - - distance: 4 - destination: 0 + - distance: 7 + destination: 8 + - distance: 7 + destination: 8 cpus: - - 6 - - 6 + - 4 + - 4 sgx_epc_sections: - sgx_epc_sections - sgx_epc_sections memory_zones: - memory_zones - memory_zones - guest_numa_id: 7 + guest_numa_id: 0 - distances: - - distance: 4 - destination: 0 - - distance: 4 - destination: 0 + - distance: 7 + destination: 8 + - distance: 7 + destination: 8 cpus: - - 6 - - 6 + - 4 + - 4 sgx_epc_sections: - sgx_epc_sections - sgx_epc_sections memory_zones: - memory_zones - memory_zones - guest_numa_id: 7 + guest_numa_id: 0 tdx: firmware: firmware rng: @@ -867,10 +915,10 @@ components: src: /dev/urandom sgx_epc: - prefault: false - size: 0 + size: 6 id: id - prefault: false - size: 0 + size: 6 id: id fs: - pci_segment: 6 @@ -897,9 +945,9 @@ components: cid: 3 platform: iommu_segments: - - 7 - - 7 - num_pci_segments: 8 + - 3 + - 3 + num_pci_segments: 3 pmem: - pci_segment: 6 mergeable: false @@ -1007,6 +1055,10 @@ components: items: $ref: '#/components/schemas/DeviceConfig' type: array + vdpa: + items: + $ref: '#/components/schemas/VdpaConfig' + type: array vsock: $ref: '#/components/schemas/VsockConfig' sgx_epc: @@ -1044,6 +1096,13 @@ components: type: integer type: array type: object + CpuFeatures: + example: + amx: true + properties: + amx: + type: boolean + type: object CpuTopology: example: dies_per_package: 5 @@ -1062,6 +1121,8 @@ components: type: object CpusConfig: example: + features: + amx: true topology: dies_per_package: 5 threads_per_core: 1 @@ -1096,6 +1157,8 @@ components: items: $ref: '#/components/schemas/CpuAffinity' type: array + features: + $ref: '#/components/schemas/CpuFeatures' required: - boot_vcpus - max_vcpus @@ -1103,9 +1166,9 @@ components: PlatformConfig: example: iommu_segments: - - 7 - - 7 - num_pci_segments: 8 + - 3 + - 3 + num_pci_segments: 3 properties: num_pci_segments: format: int16 @@ -1579,6 +1642,31 @@ components: required: - path type: object + VdpaConfig: + example: + pci_segment: 7 + path: path + num_queues: 3 + iommu: false + id: id + properties: + path: + type: string + num_queues: + default: 1 + type: integer + iommu: + default: false + type: boolean + pci_segment: + format: int16 + type: integer + id: + type: string + required: + - num_queues + - path + type: object VsockConfig: example: pci_segment: 7 @@ -1610,7 +1698,7 @@ components: SgxEpcConfig: example: prefault: false - size: 0 + size: 6 id: id properties: id: @@ -1638,8 +1726,8 @@ components: type: object NumaDistance: example: - distance: 4 - destination: 0 + distance: 7 + destination: 8 properties: destination: format: int32 @@ -1654,20 +1742,20 @@ components: NumaConfig: example: distances: - - distance: 4 - destination: 0 - - distance: 4 - destination: 0 + - distance: 7 + destination: 8 + - distance: 7 + destination: 8 cpus: - - 6 - - 6 + - 4 + - 4 sgx_epc_sections: - sgx_epc_sections - sgx_epc_sections memory_zones: - memory_zones - memory_zones - guest_numa_id: 7 + guest_numa_id: 0 properties: guest_numa_id: format: int32 diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go index 9c696060e..183bac46b 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/api_default.go @@ -1385,6 +1385,117 @@ func (a *DefaultApiService) VmAddPmemPutExecute(r ApiVmAddPmemPutRequest) (PciDe return localVarReturnValue, localVarHTTPResponse, nil } +type ApiVmAddVdpaPutRequest struct { + ctx _context.Context + ApiService *DefaultApiService + vdpaConfig *VdpaConfig +} + +// The details of the new vDPA device +func (r ApiVmAddVdpaPutRequest) VdpaConfig(vdpaConfig VdpaConfig) ApiVmAddVdpaPutRequest { + r.vdpaConfig = &vdpaConfig + return r +} + +func (r ApiVmAddVdpaPutRequest) Execute() (PciDeviceInfo, *_nethttp.Response, error) { + return r.ApiService.VmAddVdpaPutExecute(r) +} + +/* +VmAddVdpaPut Add a new vDPA device to the VM + + @param ctx _context.Context - for authentication, logging, cancellation, deadlines, tracing, etc. Passed from http.Request or context.Background(). + @return ApiVmAddVdpaPutRequest +*/ +func (a *DefaultApiService) VmAddVdpaPut(ctx _context.Context) ApiVmAddVdpaPutRequest { + return ApiVmAddVdpaPutRequest{ + ApiService: a, + ctx: ctx, + } +} + +// Execute executes the request +// @return PciDeviceInfo +func (a *DefaultApiService) VmAddVdpaPutExecute(r ApiVmAddVdpaPutRequest) (PciDeviceInfo, *_nethttp.Response, error) { + var ( + localVarHTTPMethod = _nethttp.MethodPut + localVarPostBody interface{} + localVarFormFileName string + localVarFileName string + localVarFileBytes []byte + localVarReturnValue PciDeviceInfo + ) + + localBasePath, err := a.client.cfg.ServerURLWithContext(r.ctx, "DefaultApiService.VmAddVdpaPut") + if err != nil { + return localVarReturnValue, nil, GenericOpenAPIError{error: err.Error()} + } + + localVarPath := localBasePath + "/vm.add-vdpa" + + localVarHeaderParams := make(map[string]string) + localVarQueryParams := _neturl.Values{} + localVarFormParams := _neturl.Values{} + if r.vdpaConfig == nil { + return localVarReturnValue, nil, reportError("vdpaConfig is required and must be specified") + } + + // to determine the Content-Type header + localVarHTTPContentTypes := []string{"application/json"} + + // set Content-Type header + localVarHTTPContentType := selectHeaderContentType(localVarHTTPContentTypes) + if localVarHTTPContentType != "" { + localVarHeaderParams["Content-Type"] = localVarHTTPContentType + } + + // to determine the Accept header + localVarHTTPHeaderAccepts := []string{"application/json"} + + // set Accept header + localVarHTTPHeaderAccept := selectHeaderAccept(localVarHTTPHeaderAccepts) + if localVarHTTPHeaderAccept != "" { + localVarHeaderParams["Accept"] = localVarHTTPHeaderAccept + } + // body params + localVarPostBody = r.vdpaConfig + req, err := a.client.prepareRequest(r.ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, localVarFormFileName, localVarFileName, localVarFileBytes) + if err != nil { + return localVarReturnValue, nil, err + } + + localVarHTTPResponse, err := a.client.callAPI(req) + if err != nil || localVarHTTPResponse == nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + localVarBody, err := _ioutil.ReadAll(localVarHTTPResponse.Body) + localVarHTTPResponse.Body.Close() + localVarHTTPResponse.Body = _ioutil.NopCloser(bytes.NewBuffer(localVarBody)) + if err != nil { + return localVarReturnValue, localVarHTTPResponse, err + } + + if localVarHTTPResponse.StatusCode >= 300 { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: localVarHTTPResponse.Status, + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + err = a.client.decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type")) + if err != nil { + newErr := GenericOpenAPIError{ + body: localVarBody, + error: err.Error(), + } + return localVarReturnValue, localVarHTTPResponse, newErr + } + + return localVarReturnValue, localVarHTTPResponse, nil +} + type ApiVmAddVsockPutRequest struct { ctx _context.Context ApiService *DefaultApiService diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpuFeatures.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpuFeatures.md new file mode 100644 index 000000000..8510e4083 --- /dev/null +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpuFeatures.md @@ -0,0 +1,56 @@ +# CpuFeatures + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**Amx** | Pointer to **bool** | | [optional] + +## Methods + +### NewCpuFeatures + +`func NewCpuFeatures() *CpuFeatures` + +NewCpuFeatures instantiates a new CpuFeatures object +This constructor will assign default values to properties that have it defined, +and makes sure properties required by API are set, but the set of arguments +will change when the set of required properties is changed + +### NewCpuFeaturesWithDefaults + +`func NewCpuFeaturesWithDefaults() *CpuFeatures` + +NewCpuFeaturesWithDefaults instantiates a new CpuFeatures object +This constructor will only assign default values to properties that have it defined, +but it doesn't guarantee that properties required by API are set + +### GetAmx + +`func (o *CpuFeatures) GetAmx() bool` + +GetAmx returns the Amx field if non-nil, zero value otherwise. + +### GetAmxOk + +`func (o *CpuFeatures) GetAmxOk() (*bool, bool)` + +GetAmxOk returns a tuple with the Amx field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetAmx + +`func (o *CpuFeatures) SetAmx(v bool)` + +SetAmx sets Amx field to given value. + +### HasAmx + +`func (o *CpuFeatures) HasAmx() bool` + +HasAmx returns a boolean if a field has been set. + + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpusConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpusConfig.md index 19b1a1795..8514e696f 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpusConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/CpusConfig.md @@ -9,6 +9,7 @@ Name | Type | Description | Notes **Topology** | Pointer to [**CpuTopology**](CpuTopology.md) | | [optional] **MaxPhysBits** | Pointer to **int32** | | [optional] **Affinity** | Pointer to [**[]CpuAffinity**](CpuAffinity.md) | | [optional] +**Features** | Pointer to [**CpuFeatures**](CpuFeatures.md) | | [optional] ## Methods @@ -144,6 +145,31 @@ SetAffinity sets Affinity field to given value. HasAffinity returns a boolean if a field has been set. +### GetFeatures + +`func (o *CpusConfig) GetFeatures() CpuFeatures` + +GetFeatures returns the Features field if non-nil, zero value otherwise. + +### GetFeaturesOk + +`func (o *CpusConfig) GetFeaturesOk() (*CpuFeatures, bool)` + +GetFeaturesOk returns a tuple with the Features field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetFeatures + +`func (o *CpusConfig) SetFeatures(v CpuFeatures)` + +SetFeatures sets Features field to given value. + +### HasFeatures + +`func (o *CpusConfig) HasFeatures() bool` + +HasFeatures returns a boolean if a field has been set. + [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md index c5ca050b6..b6e339559 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/DefaultApi.md @@ -18,6 +18,7 @@ Method | HTTP request | Description [**VmAddFsPut**](DefaultApi.md#VmAddFsPut) | **Put** /vm.add-fs | Add a new virtio-fs device to the VM [**VmAddNetPut**](DefaultApi.md#VmAddNetPut) | **Put** /vm.add-net | Add a new network device to the VM [**VmAddPmemPut**](DefaultApi.md#VmAddPmemPut) | **Put** /vm.add-pmem | Add a new pmem device to the VM +[**VmAddVdpaPut**](DefaultApi.md#VmAddVdpaPut) | **Put** /vm.add-vdpa | Add a new vDPA device to the VM [**VmAddVsockPut**](DefaultApi.md#VmAddVsockPut) | **Put** /vm.add-vsock | Add a new vsock device to the VM [**VmCountersGet**](DefaultApi.md#VmCountersGet) | **Get** /vm.counters | Get counters from the VM [**VmInfoGet**](DefaultApi.md#VmInfoGet) | **Get** /vm.info | Returns general information about the cloud-hypervisor Virtual Machine (VM) instance. @@ -870,6 +871,70 @@ No authorization required [[Back to README]](../README.md) +## VmAddVdpaPut + +> PciDeviceInfo VmAddVdpaPut(ctx).VdpaConfig(vdpaConfig).Execute() + +Add a new vDPA device to the VM + +### Example + +```go +package main + +import ( + "context" + "fmt" + "os" + openapiclient "./openapi" +) + +func main() { + vdpaConfig := *openapiclient.NewVdpaConfig("Path_example", int32(123)) // VdpaConfig | The details of the new vDPA device + + configuration := openapiclient.NewConfiguration() + api_client := openapiclient.NewAPIClient(configuration) + resp, r, err := api_client.DefaultApi.VmAddVdpaPut(context.Background()).VdpaConfig(vdpaConfig).Execute() + if err != nil { + fmt.Fprintf(os.Stderr, "Error when calling `DefaultApi.VmAddVdpaPut``: %v\n", err) + fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + } + // response from `VmAddVdpaPut`: PciDeviceInfo + fmt.Fprintf(os.Stdout, "Response from `DefaultApi.VmAddVdpaPut`: %v\n", resp) +} +``` + +### Path Parameters + + + +### Other Parameters + +Other parameters are passed through a pointer to a apiVmAddVdpaPutRequest struct via the builder pattern + + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + **vdpaConfig** | [**VdpaConfig**](VdpaConfig.md) | The details of the new vDPA device | + +### Return type + +[**PciDeviceInfo**](PciDeviceInfo.md) + +### Authorization + +No authorization required + +### HTTP request headers + +- **Content-Type**: application/json +- **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) +[[Back to Model list]](../README.md#documentation-for-models) +[[Back to README]](../README.md) + + ## VmAddVsockPut > PciDeviceInfo VmAddVsockPut(ctx).VsockConfig(vsockConfig).Execute() diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VdpaConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VdpaConfig.md new file mode 100644 index 000000000..bff823402 --- /dev/null +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VdpaConfig.md @@ -0,0 +1,150 @@ +# VdpaConfig + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**Path** | **string** | | +**NumQueues** | **int32** | | [default to 1] +**Iommu** | Pointer to **bool** | | [optional] [default to false] +**PciSegment** | Pointer to **int32** | | [optional] +**Id** | Pointer to **string** | | [optional] + +## Methods + +### NewVdpaConfig + +`func NewVdpaConfig(path string, numQueues int32, ) *VdpaConfig` + +NewVdpaConfig instantiates a new VdpaConfig object +This constructor will assign default values to properties that have it defined, +and makes sure properties required by API are set, but the set of arguments +will change when the set of required properties is changed + +### NewVdpaConfigWithDefaults + +`func NewVdpaConfigWithDefaults() *VdpaConfig` + +NewVdpaConfigWithDefaults instantiates a new VdpaConfig object +This constructor will only assign default values to properties that have it defined, +but it doesn't guarantee that properties required by API are set + +### GetPath + +`func (o *VdpaConfig) GetPath() string` + +GetPath returns the Path field if non-nil, zero value otherwise. + +### GetPathOk + +`func (o *VdpaConfig) GetPathOk() (*string, bool)` + +GetPathOk returns a tuple with the Path field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetPath + +`func (o *VdpaConfig) SetPath(v string)` + +SetPath sets Path field to given value. + + +### GetNumQueues + +`func (o *VdpaConfig) GetNumQueues() int32` + +GetNumQueues returns the NumQueues field if non-nil, zero value otherwise. + +### GetNumQueuesOk + +`func (o *VdpaConfig) GetNumQueuesOk() (*int32, bool)` + +GetNumQueuesOk returns a tuple with the NumQueues field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetNumQueues + +`func (o *VdpaConfig) SetNumQueues(v int32)` + +SetNumQueues sets NumQueues field to given value. + + +### GetIommu + +`func (o *VdpaConfig) GetIommu() bool` + +GetIommu returns the Iommu field if non-nil, zero value otherwise. + +### GetIommuOk + +`func (o *VdpaConfig) GetIommuOk() (*bool, bool)` + +GetIommuOk returns a tuple with the Iommu field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetIommu + +`func (o *VdpaConfig) SetIommu(v bool)` + +SetIommu sets Iommu field to given value. + +### HasIommu + +`func (o *VdpaConfig) HasIommu() bool` + +HasIommu returns a boolean if a field has been set. + +### GetPciSegment + +`func (o *VdpaConfig) GetPciSegment() int32` + +GetPciSegment returns the PciSegment field if non-nil, zero value otherwise. + +### GetPciSegmentOk + +`func (o *VdpaConfig) GetPciSegmentOk() (*int32, bool)` + +GetPciSegmentOk returns a tuple with the PciSegment field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetPciSegment + +`func (o *VdpaConfig) SetPciSegment(v int32)` + +SetPciSegment sets PciSegment field to given value. + +### HasPciSegment + +`func (o *VdpaConfig) HasPciSegment() bool` + +HasPciSegment returns a boolean if a field has been set. + +### GetId + +`func (o *VdpaConfig) GetId() string` + +GetId returns the Id field if non-nil, zero value otherwise. + +### GetIdOk + +`func (o *VdpaConfig) GetIdOk() (*string, bool)` + +GetIdOk returns a tuple with the Id field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetId + +`func (o *VdpaConfig) SetId(v string)` + +SetId sets Id field to given value. + +### HasId + +`func (o *VdpaConfig) HasId() bool` + +HasId returns a boolean if a field has been set. + + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmConfig.md b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmConfig.md index b01b81db5..d6bbae421 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmConfig.md +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/docs/VmConfig.md @@ -18,6 +18,7 @@ Name | Type | Description | Notes **Serial** | Pointer to [**ConsoleConfig**](ConsoleConfig.md) | | [optional] **Console** | Pointer to [**ConsoleConfig**](ConsoleConfig.md) | | [optional] **Devices** | Pointer to [**[]DeviceConfig**](DeviceConfig.md) | | [optional] +**Vdpa** | Pointer to [**[]VdpaConfig**](VdpaConfig.md) | | [optional] **Vsock** | Pointer to [**VsockConfig**](VsockConfig.md) | | [optional] **SgxEpc** | Pointer to [**[]SgxEpcConfig**](SgxEpcConfig.md) | | [optional] **Tdx** | Pointer to [**TdxConfig**](TdxConfig.md) | | [optional] @@ -400,6 +401,31 @@ SetDevices sets Devices field to given value. HasDevices returns a boolean if a field has been set. +### GetVdpa + +`func (o *VmConfig) GetVdpa() []VdpaConfig` + +GetVdpa returns the Vdpa field if non-nil, zero value otherwise. + +### GetVdpaOk + +`func (o *VmConfig) GetVdpaOk() (*[]VdpaConfig, bool)` + +GetVdpaOk returns a tuple with the Vdpa field if it's non-nil, zero value otherwise +and a boolean to check if the value has been set. + +### SetVdpa + +`func (o *VmConfig) SetVdpa(v []VdpaConfig)` + +SetVdpa sets Vdpa field to given value. + +### HasVdpa + +`func (o *VmConfig) HasVdpa() bool` + +HasVdpa returns a boolean if a field has been set. + ### GetVsock `func (o *VmConfig) GetVsock() VsockConfig` diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpu_features.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpu_features.go new file mode 100644 index 000000000..707b8251a --- /dev/null +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpu_features.go @@ -0,0 +1,113 @@ +/* +Cloud Hypervisor API + +Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine. + +API version: 0.3.0 +*/ + +// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT. + +package openapi + +import ( + "encoding/json" +) + +// CpuFeatures struct for CpuFeatures +type CpuFeatures struct { + Amx *bool `json:"amx,omitempty"` +} + +// NewCpuFeatures instantiates a new CpuFeatures object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewCpuFeatures() *CpuFeatures { + this := CpuFeatures{} + return &this +} + +// NewCpuFeaturesWithDefaults instantiates a new CpuFeatures object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewCpuFeaturesWithDefaults() *CpuFeatures { + this := CpuFeatures{} + return &this +} + +// GetAmx returns the Amx field value if set, zero value otherwise. +func (o *CpuFeatures) GetAmx() bool { + if o == nil || o.Amx == nil { + var ret bool + return ret + } + return *o.Amx +} + +// GetAmxOk returns a tuple with the Amx field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *CpuFeatures) GetAmxOk() (*bool, bool) { + if o == nil || o.Amx == nil { + return nil, false + } + return o.Amx, true +} + +// HasAmx returns a boolean if a field has been set. +func (o *CpuFeatures) HasAmx() bool { + if o != nil && o.Amx != nil { + return true + } + + return false +} + +// SetAmx gets a reference to the given bool and assigns it to the Amx field. +func (o *CpuFeatures) SetAmx(v bool) { + o.Amx = &v +} + +func (o CpuFeatures) MarshalJSON() ([]byte, error) { + toSerialize := map[string]interface{}{} + if o.Amx != nil { + toSerialize["amx"] = o.Amx + } + return json.Marshal(toSerialize) +} + +type NullableCpuFeatures struct { + value *CpuFeatures + isSet bool +} + +func (v NullableCpuFeatures) Get() *CpuFeatures { + return v.value +} + +func (v *NullableCpuFeatures) Set(val *CpuFeatures) { + v.value = val + v.isSet = true +} + +func (v NullableCpuFeatures) IsSet() bool { + return v.isSet +} + +func (v *NullableCpuFeatures) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableCpuFeatures(val *CpuFeatures) *NullableCpuFeatures { + return &NullableCpuFeatures{value: val, isSet: true} +} + +func (v NullableCpuFeatures) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableCpuFeatures) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpus_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpus_config.go index 52430cf00..cbcd4e034 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpus_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_cpus_config.go @@ -21,6 +21,7 @@ type CpusConfig struct { Topology *CpuTopology `json:"topology,omitempty"` MaxPhysBits *int32 `json:"max_phys_bits,omitempty"` Affinity *[]CpuAffinity `json:"affinity,omitempty"` + Features *CpuFeatures `json:"features,omitempty"` } // NewCpusConfig instantiates a new CpusConfig object @@ -190,6 +191,38 @@ func (o *CpusConfig) SetAffinity(v []CpuAffinity) { o.Affinity = &v } +// GetFeatures returns the Features field value if set, zero value otherwise. +func (o *CpusConfig) GetFeatures() CpuFeatures { + if o == nil || o.Features == nil { + var ret CpuFeatures + return ret + } + return *o.Features +} + +// GetFeaturesOk returns a tuple with the Features field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *CpusConfig) GetFeaturesOk() (*CpuFeatures, bool) { + if o == nil || o.Features == nil { + return nil, false + } + return o.Features, true +} + +// HasFeatures returns a boolean if a field has been set. +func (o *CpusConfig) HasFeatures() bool { + if o != nil && o.Features != nil { + return true + } + + return false +} + +// SetFeatures gets a reference to the given CpuFeatures and assigns it to the Features field. +func (o *CpusConfig) SetFeatures(v CpuFeatures) { + o.Features = &v +} + func (o CpusConfig) MarshalJSON() ([]byte, error) { toSerialize := map[string]interface{}{} if true { @@ -207,6 +240,9 @@ func (o CpusConfig) MarshalJSON() ([]byte, error) { if o.Affinity != nil { toSerialize["affinity"] = o.Affinity } + if o.Features != nil { + toSerialize["features"] = o.Features + } return json.Marshal(toSerialize) } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vdpa_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vdpa_config.go new file mode 100644 index 000000000..886bb4bc8 --- /dev/null +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vdpa_config.go @@ -0,0 +1,249 @@ +/* +Cloud Hypervisor API + +Local HTTP based API for managing and inspecting a cloud-hypervisor virtual machine. + +API version: 0.3.0 +*/ + +// Code generated by OpenAPI Generator (https://openapi-generator.tech); DO NOT EDIT. + +package openapi + +import ( + "encoding/json" +) + +// VdpaConfig struct for VdpaConfig +type VdpaConfig struct { + Path string `json:"path"` + NumQueues int32 `json:"num_queues"` + Iommu *bool `json:"iommu,omitempty"` + PciSegment *int32 `json:"pci_segment,omitempty"` + Id *string `json:"id,omitempty"` +} + +// NewVdpaConfig instantiates a new VdpaConfig object +// This constructor will assign default values to properties that have it defined, +// and makes sure properties required by API are set, but the set of arguments +// will change when the set of required properties is changed +func NewVdpaConfig(path string, numQueues int32) *VdpaConfig { + this := VdpaConfig{} + this.Path = path + this.NumQueues = numQueues + var iommu bool = false + this.Iommu = &iommu + return &this +} + +// NewVdpaConfigWithDefaults instantiates a new VdpaConfig object +// This constructor will only assign default values to properties that have it defined, +// but it doesn't guarantee that properties required by API are set +func NewVdpaConfigWithDefaults() *VdpaConfig { + this := VdpaConfig{} + var numQueues int32 = 1 + this.NumQueues = numQueues + var iommu bool = false + this.Iommu = &iommu + return &this +} + +// GetPath returns the Path field value +func (o *VdpaConfig) GetPath() string { + if o == nil { + var ret string + return ret + } + + return o.Path +} + +// GetPathOk returns a tuple with the Path field value +// and a boolean to check if the value has been set. +func (o *VdpaConfig) GetPathOk() (*string, bool) { + if o == nil { + return nil, false + } + return &o.Path, true +} + +// SetPath sets field value +func (o *VdpaConfig) SetPath(v string) { + o.Path = v +} + +// GetNumQueues returns the NumQueues field value +func (o *VdpaConfig) GetNumQueues() int32 { + if o == nil { + var ret int32 + return ret + } + + return o.NumQueues +} + +// GetNumQueuesOk returns a tuple with the NumQueues field value +// and a boolean to check if the value has been set. +func (o *VdpaConfig) GetNumQueuesOk() (*int32, bool) { + if o == nil { + return nil, false + } + return &o.NumQueues, true +} + +// SetNumQueues sets field value +func (o *VdpaConfig) SetNumQueues(v int32) { + o.NumQueues = v +} + +// GetIommu returns the Iommu field value if set, zero value otherwise. +func (o *VdpaConfig) GetIommu() bool { + if o == nil || o.Iommu == nil { + var ret bool + return ret + } + return *o.Iommu +} + +// GetIommuOk returns a tuple with the Iommu field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *VdpaConfig) GetIommuOk() (*bool, bool) { + if o == nil || o.Iommu == nil { + return nil, false + } + return o.Iommu, true +} + +// HasIommu returns a boolean if a field has been set. +func (o *VdpaConfig) HasIommu() bool { + if o != nil && o.Iommu != nil { + return true + } + + return false +} + +// SetIommu gets a reference to the given bool and assigns it to the Iommu field. +func (o *VdpaConfig) SetIommu(v bool) { + o.Iommu = &v +} + +// GetPciSegment returns the PciSegment field value if set, zero value otherwise. +func (o *VdpaConfig) GetPciSegment() int32 { + if o == nil || o.PciSegment == nil { + var ret int32 + return ret + } + return *o.PciSegment +} + +// GetPciSegmentOk returns a tuple with the PciSegment field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *VdpaConfig) GetPciSegmentOk() (*int32, bool) { + if o == nil || o.PciSegment == nil { + return nil, false + } + return o.PciSegment, true +} + +// HasPciSegment returns a boolean if a field has been set. +func (o *VdpaConfig) HasPciSegment() bool { + if o != nil && o.PciSegment != nil { + return true + } + + return false +} + +// SetPciSegment gets a reference to the given int32 and assigns it to the PciSegment field. +func (o *VdpaConfig) SetPciSegment(v int32) { + o.PciSegment = &v +} + +// GetId returns the Id field value if set, zero value otherwise. +func (o *VdpaConfig) GetId() string { + if o == nil || o.Id == nil { + var ret string + return ret + } + return *o.Id +} + +// GetIdOk returns a tuple with the Id field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *VdpaConfig) GetIdOk() (*string, bool) { + if o == nil || o.Id == nil { + return nil, false + } + return o.Id, true +} + +// HasId returns a boolean if a field has been set. +func (o *VdpaConfig) HasId() bool { + if o != nil && o.Id != nil { + return true + } + + return false +} + +// SetId gets a reference to the given string and assigns it to the Id field. +func (o *VdpaConfig) SetId(v string) { + o.Id = &v +} + +func (o VdpaConfig) MarshalJSON() ([]byte, error) { + toSerialize := map[string]interface{}{} + if true { + toSerialize["path"] = o.Path + } + if true { + toSerialize["num_queues"] = o.NumQueues + } + if o.Iommu != nil { + toSerialize["iommu"] = o.Iommu + } + if o.PciSegment != nil { + toSerialize["pci_segment"] = o.PciSegment + } + if o.Id != nil { + toSerialize["id"] = o.Id + } + return json.Marshal(toSerialize) +} + +type NullableVdpaConfig struct { + value *VdpaConfig + isSet bool +} + +func (v NullableVdpaConfig) Get() *VdpaConfig { + return v.value +} + +func (v *NullableVdpaConfig) Set(val *VdpaConfig) { + v.value = val + v.isSet = true +} + +func (v NullableVdpaConfig) IsSet() bool { + return v.isSet +} + +func (v *NullableVdpaConfig) Unset() { + v.value = nil + v.isSet = false +} + +func NewNullableVdpaConfig(val *VdpaConfig) *NullableVdpaConfig { + return &NullableVdpaConfig{value: val, isSet: true} +} + +func (v NullableVdpaConfig) MarshalJSON() ([]byte, error) { + return json.Marshal(v.value) +} + +func (v *NullableVdpaConfig) UnmarshalJSON(src []byte) error { + v.isSet = true + return json.Unmarshal(src, &v.value) +} diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_config.go b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_config.go index 24c2e6289..c4ad07fb0 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_config.go +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/client/model_vm_config.go @@ -30,6 +30,7 @@ type VmConfig struct { Serial *ConsoleConfig `json:"serial,omitempty"` Console *ConsoleConfig `json:"console,omitempty"` Devices *[]DeviceConfig `json:"devices,omitempty"` + Vdpa *[]VdpaConfig `json:"vdpa,omitempty"` Vsock *VsockConfig `json:"vsock,omitempty"` SgxEpc *[]SgxEpcConfig `json:"sgx_epc,omitempty"` Tdx *TdxConfig `json:"tdx,omitempty"` @@ -516,6 +517,38 @@ func (o *VmConfig) SetDevices(v []DeviceConfig) { o.Devices = &v } +// GetVdpa returns the Vdpa field value if set, zero value otherwise. +func (o *VmConfig) GetVdpa() []VdpaConfig { + if o == nil || o.Vdpa == nil { + var ret []VdpaConfig + return ret + } + return *o.Vdpa +} + +// GetVdpaOk returns a tuple with the Vdpa field value if set, nil otherwise +// and a boolean to check if the value has been set. +func (o *VmConfig) GetVdpaOk() (*[]VdpaConfig, bool) { + if o == nil || o.Vdpa == nil { + return nil, false + } + return o.Vdpa, true +} + +// HasVdpa returns a boolean if a field has been set. +func (o *VmConfig) HasVdpa() bool { + if o != nil && o.Vdpa != nil { + return true + } + + return false +} + +// SetVdpa gets a reference to the given []VdpaConfig and assigns it to the Vdpa field. +func (o *VmConfig) SetVdpa(v []VdpaConfig) { + o.Vdpa = &v +} + // GetVsock returns the Vsock field value if set, zero value otherwise. func (o *VmConfig) GetVsock() VsockConfig { if o == nil || o.Vsock == nil { @@ -784,6 +817,9 @@ func (o VmConfig) MarshalJSON() ([]byte, error) { if o.Devices != nil { toSerialize["devices"] = o.Devices } + if o.Vdpa != nil { + toSerialize["vdpa"] = o.Vdpa + } if o.Vsock != nil { toSerialize["vsock"] = o.Vsock } diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml index c9ecd399e..8861f1e71 100644 --- a/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml +++ b/src/runtime/virtcontainers/pkg/cloud-hypervisor/cloud-hypervisor.yaml @@ -325,7 +325,28 @@ paths: description: The new device was successfully (cold) added to the VM instance. 500: description: The new device could not be added to the VM instance. - + + /vm.add-vdpa: + put: + summary: Add a new vDPA device to the VM + requestBody: + description: The details of the new vDPA device + content: + application/json: + schema: + $ref: '#/components/schemas/VdpaConfig' + required: true + responses: + 200: + description: The new vDPA device was successfully added to the VM instance. + content: + application/json: + schema: + $ref: '#/components/schemas/PciDeviceInfo' + 204: + description: The new vDPA device was successfully (cold) added to the VM instance. + 500: + description: The new vDPA device could not be added to the VM instance. /vm.snapshot: put: @@ -505,6 +526,10 @@ components: type: array items: $ref: '#/components/schemas/DeviceConfig' + vdpa: + type: array + items: + $ref: '#/components/schemas/VdpaConfig' vsock: $ref: '#/components/schemas/VsockConfig' sgx_epc: @@ -537,6 +562,12 @@ components: items: type: integer + CpuFeatures: + type: object + properties: + amx: + type: boolean + CpuTopology: type: object properties: @@ -571,6 +602,8 @@ components: type: array items: $ref: '#/components/schemas/CpuAffinity' + features: + $ref: '#/components/schemas/CpuFeatures' PlatformConfig: type: object @@ -921,6 +954,26 @@ components: id: type: string + VdpaConfig: + required: + - path + - num_queues + type: object + properties: + path: + type: string + num_queues: + type: integer + default: 1 + iommu: + type: boolean + default: false + pci_segment: + type: integer + format: int16 + id: + type: string + VsockConfig: required: - cid From 7b2ff0264710dbb3cb1106cd1568408ebb720532 Mon Sep 17 00:00:00 2001 From: Francesco Giudici Date: Wed, 13 Apr 2022 17:46:48 +0200 Subject: [PATCH 14/15] kata-monitor: add a README file Fixes: #3704 Signed-off-by: Francesco Giudici --- docs/design/kata-2-0-metrics.md | 1 + src/runtime/README.md | 1 + src/runtime/cmd/kata-monitor/README.md | 68 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 src/runtime/cmd/kata-monitor/README.md diff --git a/docs/design/kata-2-0-metrics.md b/docs/design/kata-2-0-metrics.md index 2116ee90a..e8923a454 100644 --- a/docs/design/kata-2-0-metrics.md +++ b/docs/design/kata-2-0-metrics.md @@ -51,6 +51,7 @@ The `kata-monitor` management agent should be started on each node where the Kat > **Note**: a *node* running Kata containers will be either a single host system or a worker node belonging to a K8s cluster capable of running Kata pods. - Aggregate sandbox metrics running on the node, adding the `sandbox_id` label to them. +- Attach the additional `cri_uid`, `cri_name` and `cri_namespace` labels to the sandbox metrics, tracking the `uid`, `name` and `namespace` Kubernetes pod metadata. - Expose a new Prometheus target, allowing all node metrics coming from the Kata shim to be collected by Prometheus indirectly. This simplifies the targets count in Prometheus and avoids exposing shim's metrics by `ip:port`. Only one `kata-monitor` process runs in each node. diff --git a/src/runtime/README.md b/src/runtime/README.md index 0cd77b995..75febbe9d 100644 --- a/src/runtime/README.md +++ b/src/runtime/README.md @@ -10,6 +10,7 @@ This repository contains the following components: |-|-| | `containerd-shim-kata-v2` | The [shimv2 runtime](../../docs/design/architecture/README.md#runtime) | | `kata-runtime` | [utility program](../../docs/design/architecture/README.md#utility-program) | +| `kata-monitor` | [metrics collector daemon](cmd/kata-monitor/README.md) | For details of the other Kata Containers repositories, see the [repository summary](https://github.com/kata-containers/kata-containers). diff --git a/src/runtime/cmd/kata-monitor/README.md b/src/runtime/cmd/kata-monitor/README.md new file mode 100644 index 000000000..81ae2ee0a --- /dev/null +++ b/src/runtime/cmd/kata-monitor/README.md @@ -0,0 +1,68 @@ +# Kata monitor + +## Overview +`kata-monitor` is a daemon able to collect and expose metrics related to all the Kata Containers workloads running on the same host. +Once started, it detects all the running Kata Containers runtimes (`containerd-shim-kata-v2`) in the system and exposes few http endpoints to allow the retrieval of the available data. +The main endpoint is the `/metrics` one which aggregates metrics from all the kata workloads. +Available metrics include: + * Kata runtime metrics + * Kata agent metrics + * Kata guest OS metrics + * Hypervisor metrics + * Firecracker metrics + * Kata monitor metrics + +All the provided metrics are in Prometheus format. While `kata-monitor` can be used as a standalone daemon on any host running Kata Containers workloads and can be used for retrieving profiling data from the running Kata runtimes, its main expected usage is to be deployed as a DaemonSet on a Kubernetes cluster: there Prometheus should scrape the metrics from the kata-monitor endpoints. +For more information on the Kata Containers metrics architecture and a detailed list of the available metrics provided by Kata monitor check the [Kata 2.0 Metrics Design](../../../../docs/design/kata-2-0-metrics.md) document. + +## Usage +Each `kata-monitor` instance detects and monitors the Kata Container workloads running on the same node. + +### Kata monitor arguments +The `kata-monitor` binary accepts the following arguments: + +* `--listen-address` _IP:PORT_ +* `--runtime-enpoint` _PATH_TO_THE_CONTAINER_MANAGER_CRI_INTERFACE_ +* `--log-level` _[ trace | debug | info | warn | error | fatal | panic ]_ + +The **listen-address** specifies the IP and TCP port where the kata-monitor HTTP endpoints will be exposed. It defaults to `127.0.0.1:8090`. + +The **runtime-endpoint** is the CRI of a CRI compliant container manager: it will be used to retrieve the CRI `PodSandboxMetadata` (`uid`, `name` and `namespace`) which will be attached to the Kata metrics through the labels `cri_uid`, `cri_name` and `cri_namespace`. It defaults to the containerd socket: `/run/containerd/containerd.sock`. + +The **log-level** allows the chose how verbose the logs should be. The default is `info`. +### Kata monitor HTTP endpoints +`kata-monitor` exposes the following endpoints: + * `/metrics` : get Kata sandboxes metrics. + * `/sandboxes` : list all the Kata sandboxes running on the host. + * `/agent-url` : Get the agent URL of a Kata sandbox. + * `/debug/vars` : Internal data of the Kata runtime shim. + * `/debug/pprof/` : Golang profiling data of the Kata runtime shim: index page. + * `/debug/pprof/cmdline` : Golang profiling data of the Kata runtime shim: `cmdline` endpoint. + * `/debug/pprof/profile` : Golang profiling data of the Kata runtime shim: `profile` endpoint (CPU profiling). + * `/debug/pprof/symbol` : Golang profiling data of the Kata runtime shim: `symbol` endpoint. + * `/debug/pprof/trace` : Golang profiling data of the Kata runtime shim: `trace` endpoint. + +**NOTE: The debug endpoints are available only if the [Kata Containers configuration file](https://github.com/kata-containers/kata-containers/blob/9d5b03a1b70bbd175237ec4b9f821d6ccee0a1f6/src/runtime/config/configuration-qemu.toml.in#L590-L592) includes** `enable_pprof = true` **in the** `[runtime]` **section**. + +The `/sandboxes` endpoint lists the _sandbox ID_ of all the detected Kata runtimes. If accessed via a web browser, it provides html links to the endpoints available for each sandbox. + +In order to retrieve data for a specific Kata workload, the _sandbox ID_ should be passed in the query string using the _sandbox_ key. The `/agent-url`, and all the `/debug/`* endpoints require `sandbox_id` to be specified in the query string. +
+#### Examples +Retrieve the IDs of the available sandboxes: +```bash +$ curl 127.0.0.1:8090/sandboxes +``` +output: +``` +6fcf0a90b01e90d8747177aa466c3462d02e02a878bc393649df83d4c314af0c +df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343 +``` +Retrieve the `agent-url` of the sandbox with ID _df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343_: +```bash +$ curl 127.0.0.1:8090/agent-url?sandbox=df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343 +``` +output: +``` +vsock://830455376:1024 +``` From 2256bcb6ab871b2d3b276c20406eeb4f71f3c568 Mon Sep 17 00:00:00 2001 From: Garrett Mahin Date: Wed, 13 Apr 2022 15:45:51 -0500 Subject: [PATCH 15/15] rustjail: Add tests for root_grpc_to_oci Add test coverage for root_grpc_to_oci in rustjail/src/lib.rs Fixes: #4095 Signed-off-by: Garrett Mahin --- src/agent/rustjail/src/lib.rs | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs index 5b98080c3..3861bce48 100644 --- a/src/agent/rustjail/src/lib.rs +++ b/src/agent/rustjail/src/lib.rs @@ -680,4 +680,61 @@ mod tests { assert_eq!(d.result, result, "{}", msg); } } + + #[test] + fn test_root_grpc_to_oci() { + #[derive(Debug)] + struct TestData { + grpcroot: grpc::Root, + result: oci::Root, + } + + let tests = &[ + TestData { + // Default fields + grpcroot: grpc::Root { + ..Default::default() + }, + result: oci::Root { + ..Default::default() + }, + }, + TestData { + // Specified fields, readonly false + grpcroot: grpc::Root { + Path: String::from("path"), + Readonly: false, + ..Default::default() + }, + result: oci::Root { + path: String::from("path"), + readonly: false, + ..Default::default() + }, + }, + TestData { + // Specified fields, readonly true + grpcroot: grpc::Root { + Path: String::from("path"), + Readonly: true, + ..Default::default() + }, + result: oci::Root { + path: String::from("path"), + readonly: true, + ..Default::default() + }, + }, + ]; + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let result = root_grpc_to_oci(&d.grpcroot); + + let msg = format!("{}, result: {:?}", msg, result); + + assert_eq!(d.result, result, "{}", msg); + } + } }